examples updates

aimclub · Apr 8, 2024 · 787135d · 787135d
1 parent 2d04717
commit 787135d
Show file tree

Hide file tree

Showing 22 changed files with 132 additions and 85 deletions.
diff --git a/examples/automl_example/api_example/time_series/ts_forecasting/ts_forecasting_example.py b/examples/automl_example/api_example/time_series/ts_forecasting/ts_forecasting_example.py
@@ -1,38 +1,69 @@
+import pandas as pd
 from fedot.core.pipelines.pipeline_builder import PipelineBuilder
 
 from fedot_ind.api.main import FedotIndustrial
+from fedot_ind.api.utils.path_lib import PROJECT_PATH
+from fedot_ind.core.metrics.metrics_implementation import calculate_forecasting_metric
+from fedot_ind.core.repository.constanst_repository import M4_FORECASTING_BENCH
 from fedot_ind.tools.loader import DataLoader
 
 if __name__ == "__main__":
-    dataset_name = 'D1317'
+
+    #dataset_name = 'D1317'
     benchmark = 'M4'
     horizon = 14
     finetune = False
-    initial_assumption = PipelineBuilder().add_node('eigen_basis',
-                                                    params={'low_rank_approximation': False,
-                                                            'rank_regularization': 'explained_dispersion'}).add_node(
-        'ar')
-
-    industrial = FedotIndustrial(problem='ts_forecasting',
-                                 metric='rmse',
-                                 task_params={'forecast_length': horizon},
-                                 timeout=5,
-                                 with_tuning=False,
-                                 initial_assumption=initial_assumption,
-                                 n_jobs=2,
-                                 logging_level=10)
-
-    train_data, test_data = DataLoader(dataset_name=dataset_name).load_forecast_data(folder=benchmark)
-
-    if finetune:
-        model = industrial.finetune(train_data)
-    else:
-        model = industrial.fit(train_data)
-
-    labels = industrial.predict(test_data)
-    probs = industrial.predict_proba(test_data)
-    metrics = industrial.get_metrics(target=test_data[1],
-                                     rounding_order=3,
-                                     metric_names=['f1', 'accuracy', 'precision', 'roc_auc'])
-    print(metrics)
-    _ = 1
+    for dataset_name in M4_FORECASTING_BENCH:
+        try:
+            autogluon = PROJECT_PATH + f'/benchmark/results/benchmark_results/autogluon/' \
+                                       f'{dataset_name}_{horizon}_forecast_vs_actual.csv'
+            n_beats = PROJECT_PATH + f'/benchmark/results/benchmark_results/nbeats/' \
+                                     f'{dataset_name}_{horizon}_forecast_vs_actual.csv'
+            n_beats = pd.read_csv(n_beats)
+            autogluon = pd.read_csv(autogluon)
+
+            n_beats_forecast = calculate_forecasting_metric(target=n_beats['value'].values,
+                                                            labels=n_beats['predict'].values)
+            autogluon_forecast = calculate_forecasting_metric(target=autogluon['value'].values,
+                                                              labels=autogluon['predict'].values)
+
+            initial_assumption = PipelineBuilder().add_node('eigen_basis',
+                                                            params={'low_rank_approximation': False,
+                                                                    'rank_regularization': 'explained_dispersion'}).add_node(
+                'ar')
+            industrial = FedotIndustrial(problem='ts_forecasting',
+                                         metric='rmse',
+                                         task_params={'forecast_length': horizon},
+                                         timeout=5,
+                                         with_tuning=False,
+                                         initial_assumption=initial_assumption,
+                                         n_jobs=2,
+                                         logging_level=30)
+
+            train_data, _ = DataLoader(dataset_name=dataset_name).load_forecast_data(folder=benchmark)
+
+            if finetune:
+                model = industrial.finetune(train_data)
+            else:
+                model = industrial.fit(train_data)
+
+            labels = industrial.predict(train_data)
+            metrics = industrial.get_metrics(target=train_data.values[-horizon:].flatten(),
+                                             metric_names=('smape', 'rmse', 'median_absolute_error'))
+            industrial.save_best_model()
+            forecast = pd.DataFrame([labels,
+                                     train_data.values[-horizon:].flatten(),
+                                     autogluon['predict'].values,
+                                     n_beats['predict'].values]).T
+            forecast.columns = ['industrial', 'target',
+                                'AG',
+                                'NBEATS']
+            metrics_comprasion = pd.concat([metrics, autogluon_forecast, n_beats_forecast]).T
+            metrics_comprasion.columns = ['industrial',
+                                          'AG',
+                                          'NBEATS']
+            forecast.to_csv(f'./{dataset_name}_forecast.csv')
+            metrics_comprasion.to_csv(f'./{dataset_name}_metrics.csv')
+        except Exception:
+            print(f'Skip {dataset_name}')
+
diff --git a/examples/automl_example/api_example/time_series/ts_regression/ts_regression_example.py b/examples/automl_example/api_example/time_series/ts_regression/ts_regression_example.py
@@ -22,9 +22,8 @@
         model = industrial.fit(train_data)
 
     labels = industrial.predict(test_data)
-    probs = industrial.predict_proba(test_data)
     metrics = industrial.get_metrics(target=test_data[1],
                                      rounding_order=3,
                                      metric_names=('r2', 'rmse', 'mae'))
     print(metrics)
-    _ = 1
+
diff --git a/examples/real_world_examples/industrial_examples/economic_analysis/requirements.txt b/examples/real_world_examples/industrial_examples/economic_analysis/requirements.txt
diff --git a/examples/real_world_examples/industrial_examples/economic_analysis/ts_regression/__init__.py b/examples/real_world_examples/industrial_examples/economic_analysis/ts_regression/__init__.py
diff --git a/...es/economic_analysis/oil_gas_prices.ipynb → ...alysis/ts_regression/oil_gas_prices.ipynb b/...es/economic_analysis/oil_gas_prices.ipynb → ...alysis/ts_regression/oil_gas_prices.ipynb
diff --git a/...s/economic_analysis/scoring_prediction.py → ...lysis/ts_regression/scoring_prediction.py b/...s/economic_analysis/scoring_prediction.py → ...lysis/ts_regression/scoring_prediction.py
diff --git a/...mples/economic_analysis/scoring_train.csv → ..._analysis/ts_regression/scoring_train.csv b/...mples/economic_analysis/scoring_train.csv → ..._analysis/ts_regression/scoring_train.csv
diff --git a/...conomic_analysis/scroing_prediction.ipynb → ...is/ts_regression/scroing_prediction.ipynb b/...conomic_analysis/scroing_prediction.ipynb → ...is/ts_regression/scroing_prediction.ipynb
diff --git a/examples/real_world_examples/industrial_examples/energy_monitoring/ts_regression/__init__.py b/examples/real_world_examples/industrial_examples/energy_monitoring/ts_regression/__init__.py
diff --git a/...ding_energy_consumption_electricity.ipynb → ...ding_energy_consumption_electricity.ipynb b/...ding_energy_consumption_electricity.ipynb → ...ding_energy_consumption_electricity.ipynb
diff --git a/...uilding_energy_consumption_hotwater.ipynb → ...uilding_energy_consumption_hotwater.ipynb b/...uilding_energy_consumption_hotwater.ipynb → ...uilding_energy_consumption_hotwater.ipynb
diff --git a/...g/building_energy_consumption_steam.ipynb → ...n/building_energy_consumption_steam.ipynb b/...g/building_energy_consumption_steam.ipynb → ...n/building_energy_consumption_steam.ipynb
diff --git a/...ples/real_world_examples/industrial_examples/sentiment_analysis/ts_regression/__init__.py b/...ples/real_world_examples/industrial_examples/sentiment_analysis/ts_regression/__init__.py
diff --git a/...sentiment_analysis/bitcoin_analysis.ipynb → ...ysis/ts_regression/bitcoin_analysis.ipynb b/...sentiment_analysis/bitcoin_analysis.ipynb → ...ysis/ts_regression/bitcoin_analysis.ipynb
diff --git a/...entiment_analysis/ethereum_analysis.ipynb → ...sis/ts_regression/ethereum_analysis.ipynb b/...entiment_analysis/ethereum_analysis.ipynb → ...sis/ts_regression/ethereum_analysis.ipynb
diff --git a/fedot_ind/core/metrics/metrics_implementation.py b/fedot_ind/core/metrics/metrics_implementation.py
@@ -9,6 +9,9 @@
 from sklearn.metrics import d2_absolute_error_score, explained_variance_score, max_error, median_absolute_error
 
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
+from sktime.performance_metrics.forecasting import mean_absolute_scaled_error
+
+import numpy as np
 
 
 class ParetoMetrics:
@@ -70,8 +73,8 @@ def metric(self) -> float:
 class SMAPE(QualityMetric):
     def metric(self):
         return 1 / len(self.predicted_labels) \
-            * np.sum(2 * np.abs(self.target - self.predicted_labels) / (np.abs(self.predicted_labels)
-                                                                        + np.abs(self.target)) * 100)
+               * np.sum(2 * np.abs(self.target - self.predicted_labels) / (np.abs(self.predicted_labels)
+                                                                           + np.abs(self.target)) * 100)
 
 
 class MSE(QualityMetric):
@@ -167,6 +170,14 @@ def metric(self) -> float:
         return accuracy_score(y_true=self.target, y_pred=self.predicted_labels)
 
 
+def MASE(A, F, y_train):
+    return mean_absolute_scaled_error(A, F, y_train=y_train)
+
+
+def SMAPE(a, f, _=None):
+    return 1 / len(a) * np.sum(2 * np.abs(f - a) / (np.abs(a) + np.abs(f)) * 100)
+
+
 def calculate_regression_metric(target,
                                 labels,
                                 rounding_order=3,
@@ -194,6 +205,30 @@ def rmse(y_true, y_pred):
     return df.round(rounding_order)
 
 
+def calculate_forecasting_metric(target,
+                                 labels,
+                                 rounding_order=3,
+                                 metric_names=('smape', 'rmse', 'median_absolute_error'),
+                                 **kwargs):
+    target = target.astype(float)
+
+    def rmse(y_true, y_pred):
+        return np.sqrt(mean_squared_error(y_true, y_pred))
+
+    metric_dict = {
+        'rmse': rmse,
+        'mae': mean_absolute_error,
+        'median_absolute_error': median_absolute_error,
+        'smape': SMAPE,
+        'mase': MASE
+    }
+
+    df = pd.DataFrame({name: func(target, labels) for name, func in metric_dict.items()
+                       if name in metric_names},
+                      index=[0])
+    return df.round(rounding_order)
+
+
 def calculate_classification_metric(target,
                                     labels,
                                     probs,

diff --git a/fedot_ind/core/operation/interfaces/industrial_preprocessing_strategy.py b/fedot_ind/core/operation/interfaces/industrial_preprocessing_strategy.py
@@ -324,6 +324,8 @@ def __init__(self, operation_type: str, params: Optional[OperationParameters] =
         params = IndustrialOperationParameters().from_params(operation_type, params) if params \
             else IndustrialOperationParameters().from_operation_type(operation_type)
         super().__init__(operation_type, params)
+        self.multi_dim_dispatcher.concat_func = np.vstack
+        self.ensemble_func = np.sum
 
     def fit(self, train_data: InputData):
         train_data = self.multi_dim_dispatcher._convert_input_data(train_data)
@@ -334,15 +336,23 @@ def predict(self, trained_operation,
                 output_mode: str = 'default'):
         converted_predict_data = self.multi_dim_dispatcher._convert_input_data(
             predict_data)
-        return self.multi_dim_dispatcher.predict(trained_operation, converted_predict_data, output_mode=output_mode)
+        predict_output = self.multi_dim_dispatcher.predict(trained_operation, converted_predict_data,
+                                                           output_mode=output_mode)
+        predict_output.predict = np.reshape(predict_output.predict, (len(trained_operation), -1))
+        predict_output.predict = self.ensemble_func(predict_output.predict, axis=0)
+        return predict_output
 
     def predict_for_fit(self, trained_operation,
                         predict_data: InputData,
                         output_mode: str = 'default') -> OutputData:
         converted_predict_data = self.multi_dim_dispatcher._convert_input_data(
             predict_data)
-        return self.multi_dim_dispatcher.predict_for_fit(trained_operation, converted_predict_data,
-                                                         output_mode=output_mode)
+        predict_output = self.multi_dim_dispatcher.predict_for_fit(trained_operation,
+                                                                   converted_predict_data,
+                                                                   output_mode=output_mode)
+        predict_output.predict = np.reshape(predict_output.predict, (len(trained_operation), -1))
+        predict_output.predict = self.ensemble_func(predict_output.predict, axis=0)
+        return predict_output
 
 
 class IndustrialClassificationPreprocessingStrategy(IndustrialCustomPreprocessingStrategy):

diff --git a/fedot_ind/core/repository/constanst_repository.py b/fedot_ind/core/repository/constanst_repository.py
@@ -14,7 +14,8 @@
 from torch import nn
 from golem.core.tuning.simultaneous import SimultaneousTuner
 from golem.core.tuning.sequential import SequentialTuner
-from fedot_ind.core.metrics.metrics_implementation import calculate_classification_metric, calculate_regression_metric
+from fedot_ind.core.metrics.metrics_implementation import calculate_classification_metric, calculate_regression_metric, \
+    calculate_forecasting_metric
 from fedot_ind.core.models.nn.network_modules.losses import CenterLoss, CenterPlusLoss, ExpWeightedLoss, FocalLoss, \
     HuberLoss, LogCoshLoss, MaskedLossWrapper, RMSELoss, SMAPELoss, TweedieLoss
 from fedot_ind.core.models.quantile.stat_features import autocorrelation, ben_corr, crest_factor, energy, \
@@ -193,8 +194,10 @@ class FedotOperationConstant(Enum):
                                                         with_tuning=True
                                                         )
     FEDOT_GET_METRICS = {'regression': calculate_regression_metric,
+                         'ts_forecasting': calculate_forecasting_metric,
                          'classification': calculate_classification_metric}
     FEDOT_TUNING_METRICS = {'classification': ClassificationMetricsEnum.accuracy,
+                            'ts_forecasting': RegressionMetricsEnum.RMSE,
                             'regression': RegressionMetricsEnum.RMSE}
     FEDOT_TUNER_STRATEGY = {
         'sequential': partial(SequentialTuner, inverse_node_order=True),
@@ -229,7 +232,7 @@ class FedotOperationConstant(Enum):
         'classification': PipelineBuilder().add_node('channel_filtration').add_node('quantile_extractor').add_node(
             'logit'),
         'regression': PipelineBuilder().add_node('channel_filtration').add_node('quantile_extractor').add_node('treg'),
-        'ts_forecasting': PipelineBuilder().add_node('ssa_forecaster')
+        'ts_forecasting': PipelineBuilder().add_node('ar')
     }
 
     FEDOT_ENSEMBLE_ASSUMPTIONS = {

diff --git a/fedot_ind/core/repository/data/industrial_data_operation_repository.json b/fedot_ind/core/repository/data/industrial_data_operation_repository.json
@@ -169,7 +169,9 @@
         "fast_train"
       ],
       "tags": [
-        "basis"
+        "basis",
+		  "non_lagged",
+		  "ts_to_ts"
       ]
     },
     "wavelet_basis": {
@@ -178,7 +180,7 @@
         "fast_train"
       ],
       "tags": [
-        "basis"
+        "basis","non_applicable_for_ts"
       ]
     },
     "fourier_basis": {
@@ -187,7 +189,7 @@
         "fast_train"
       ],
       "tags": [
-        "basis"
+        "basis","non_applicable_for_ts"
       ]
     },
     "quantile_extractor": {
@@ -205,7 +207,7 @@
         "fast_train"
       ],
       "tags": [
-        "extractor"
+        "extractor","non_applicable_for_ts"
       ]
     },
     "recurrence_extractor": {
@@ -214,7 +216,7 @@
         "fast_train"
       ],
       "tags": [
-        "extractor"
+        "extractor","non_applicable_for_ts"
       ]
 	},
       "topological_features": {
@@ -231,7 +233,7 @@
         "fast_train"
       ],
       "tags": [
-        "extractor"
+        "extractor","non_applicable_for_ts"
       ]
     },
     "chronos_extractor": {
@@ -240,7 +242,7 @@
         "fast_train"
       ],
       "tags": [
-        "extractor"
+        "extractor","non_applicable_for_ts"
       ]
     },
     "channel_filtration": {
@@ -249,7 +251,7 @@
         "fast_train"
       ],
       "tags": [
-        "extractor"
+        "extractor","non_applicable_for_ts"
       ]
     },
     "cat_features": {
@@ -258,7 +260,7 @@
         "fast_train"
       ],
       "tags": [
-        "cat_features"
+        "cat_features","non_applicable_for_ts"
       ]
     },
     "data_source_img": {

diff --git a/fedot_ind/core/repository/initializer_industrial_models.py b/fedot_ind/core/repository/initializer_industrial_models.py
@@ -81,7 +81,7 @@ def setup_repository(self):
                 transform_lagged_for_fit)
         setattr(TsSmoothingImplementation, 'transform', transform_smoothing)
 
-        # class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline)
+        class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline)
         MutationStrengthEnum = MutationStrengthEnumIndustrial
         return OperationTypesRepository
 
@@ -108,7 +108,6 @@ def __enter__(self):
                 get_industrial_search_space)
         setattr(ApiComposer, "_get_default_mutations",
                 _get_default_industrial_mutations)
-        class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline)
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         """

diff --git a/fedot_ind/core/repository/model_repository.py b/fedot_ind/core/repository/model_repository.py
@@ -181,7 +181,7 @@ class AtomizedModel(Enum):
         'ets': ExpSmoothingImplementation,
         'cgru': CGRUImplementation,
         'glm': GLMImplementation,
-        'locf': RepeatLastValueImplementation,
+        #'locf': RepeatLastValueImplementation,
         #'ssa_forecaster': SSAForecasterImplementation
     }