From e63ba4931d34f23323f76a492655f703da5cb007 Mon Sep 17 00:00:00 2001 From: Valerii Pokrovskii <32017472+valer1435@users.noreply.github.com> Date: Tue, 12 Sep 2023 16:24:00 +0300 Subject: [PATCH] Hotfix after ssa (#82) Return basis in transformation, Move ssa into models --- .../architecture/pipelines/classification.py | 2 +- .../architecture/settings/pipeline_factory.py | 4 +- .../core/models/signal/SignalExtractor.py | 2 +- .../ts_forecasting}/__init__.py | 0 .../ts_forecasting/data_driven_ts.py | 0 .../implementation/basis/data_driven.py | 0 ..._decomposition_for_forecasting_strategy.py | 2 +- .../industrial_preprocessing_strategy.py | 4 +- .../basis/__init__.py | 0 .../basis/abstract_basis.py | 0 .../transformation/basis/data_driven.py | 88 +++++++++++++++---- .../basis/fourier.py | 2 +- .../basis/wavelet.py | 2 +- .../core/operation/ts_forecasting/__init__.py | 0 14 files changed, 79 insertions(+), 27 deletions(-) rename fedot_ind/core/{operation/implementation => models/ts_forecasting}/__init__.py (100%) rename fedot_ind/core/{operation => models}/ts_forecasting/data_driven_ts.py (100%) delete mode 100644 fedot_ind/core/operation/implementation/basis/data_driven.py rename fedot_ind/core/operation/{implementation => transformation}/basis/__init__.py (100%) rename fedot_ind/core/operation/{implementation => transformation}/basis/abstract_basis.py (100%) rename fedot_ind/core/operation/{implementation => transformation}/basis/fourier.py (96%) rename fedot_ind/core/operation/{implementation => transformation}/basis/wavelet.py (97%) delete mode 100644 fedot_ind/core/operation/ts_forecasting/__init__.py diff --git a/fedot_ind/core/architecture/pipelines/classification.py b/fedot_ind/core/architecture/pipelines/classification.py index 9dbf523af..0d34c42de 100644 --- a/fedot_ind/core/architecture/pipelines/classification.py +++ b/fedot_ind/core/architecture/pipelines/classification.py @@ -2,7 +2,7 @@ from pymonad.either import Right from fedot_ind.core.architecture.pipelines.abstract_pipeline import AbstractPipelines from fedot_ind.core.architecture.preprocessing.DatasetLoader import DataLoader -from fedot_ind.core.operation.implementation.basis.data_driven import DataDrivenBasisImplementation +from fedot_ind.core.operation.transformation.basis.data_driven import DataDrivenBasisImplementation from functools import partial diff --git a/fedot_ind/core/architecture/settings/pipeline_factory.py b/fedot_ind/core/architecture/settings/pipeline_factory.py index 2e563a6fa..15c6ecc04 100644 --- a/fedot_ind/core/architecture/settings/pipeline_factory.py +++ b/fedot_ind/core/architecture/settings/pipeline_factory.py @@ -7,9 +7,9 @@ from fedot_ind.core.models.recurrence.RecurrenceExtractor import RecurrenceExtractor from fedot_ind.core.models.signal.SignalExtractor import SignalExtractor from fedot_ind.core.models.topological.TopologicalExtractor import TopologicalExtractor -from fedot_ind.core.operation.implementation.basis.fourier import FourierBasisImplementation -from fedot_ind.core.operation.implementation.basis.wavelet import WaveletBasisImplementation from fedot_ind.core.operation.transformation.basis.data_driven import DataDrivenBasisImplementation +from fedot_ind.core.operation.transformation.basis.fourier import FourierBasisImplementation +from fedot_ind.core.operation.transformation.basis.wavelet import WaveletBasisImplementation diff --git a/fedot_ind/core/models/signal/SignalExtractor.py b/fedot_ind/core/models/signal/SignalExtractor.py index f5a4672b4..d91885da9 100644 --- a/fedot_ind/core/models/signal/SignalExtractor.py +++ b/fedot_ind/core/models/signal/SignalExtractor.py @@ -8,7 +8,7 @@ from fedot_ind.core.metrics.metrics_implementation import * from fedot_ind.core.models.WindowedFeaturesExtractor import WindowedFeatureExtractor from fedot_ind.core.models.quantile.quantile_extractor import QuantileExtractor -from fedot_ind.core.operation.implementation.basis.wavelet import WaveletBasisImplementation +from fedot_ind.core.operation.transformation.basis.wavelet import WaveletBasisImplementation class SignalExtractor(WindowedFeatureExtractor): diff --git a/fedot_ind/core/operation/implementation/__init__.py b/fedot_ind/core/models/ts_forecasting/__init__.py similarity index 100% rename from fedot_ind/core/operation/implementation/__init__.py rename to fedot_ind/core/models/ts_forecasting/__init__.py diff --git a/fedot_ind/core/operation/ts_forecasting/data_driven_ts.py b/fedot_ind/core/models/ts_forecasting/data_driven_ts.py similarity index 100% rename from fedot_ind/core/operation/ts_forecasting/data_driven_ts.py rename to fedot_ind/core/models/ts_forecasting/data_driven_ts.py diff --git a/fedot_ind/core/operation/implementation/basis/data_driven.py b/fedot_ind/core/operation/implementation/basis/data_driven.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/fedot_ind/core/operation/interfaces/industrial_decomposition_for_forecasting_strategy.py b/fedot_ind/core/operation/interfaces/industrial_decomposition_for_forecasting_strategy.py index 9b6981f10..2ac2e5a4a 100644 --- a/fedot_ind/core/operation/interfaces/industrial_decomposition_for_forecasting_strategy.py +++ b/fedot_ind/core/operation/interfaces/industrial_decomposition_for_forecasting_strategy.py @@ -3,7 +3,7 @@ from fedot.core.operations.evaluation.time_series import FedotTsForecastingStrategy from fedot.core.operations.operation_parameters import OperationParameters -from fedot_ind.core.operation.ts_forecasting.data_driven_ts import DataDrivenForForecastingBasisImplementation +from fedot_ind.core.models.ts_forecasting.data_driven_ts import DataDrivenForForecastingBasisImplementation from fedot_ind.core.repository.IndustrialOperationParameters import IndustrialOperationParameters diff --git a/fedot_ind/core/operation/interfaces/industrial_preprocessing_strategy.py b/fedot_ind/core/operation/interfaces/industrial_preprocessing_strategy.py index 31ae5ed39..732062e54 100644 --- a/fedot_ind/core/operation/interfaces/industrial_preprocessing_strategy.py +++ b/fedot_ind/core/operation/interfaces/industrial_preprocessing_strategy.py @@ -9,8 +9,8 @@ from fedot_ind.core.models.topological.TopologicalExtractor import TopologicalExtractor from fedot_ind.core.operation.dummy.dummy_operation import DummyOperation -from fedot_ind.core.operation.implementation.basis.fourier import FourierBasisImplementation -from fedot_ind.core.operation.implementation.basis.wavelet import WaveletBasisImplementation +from fedot_ind.core.operation.transformation.basis.fourier import FourierBasisImplementation +from fedot_ind.core.operation.transformation.basis.wavelet import WaveletBasisImplementation from fedot_ind.core.operation.transformation.basis.data_driven import DataDrivenBasisImplementation from fedot_ind.core.repository.IndustrialOperationParameters import IndustrialOperationParameters diff --git a/fedot_ind/core/operation/implementation/basis/__init__.py b/fedot_ind/core/operation/transformation/basis/__init__.py similarity index 100% rename from fedot_ind/core/operation/implementation/basis/__init__.py rename to fedot_ind/core/operation/transformation/basis/__init__.py diff --git a/fedot_ind/core/operation/implementation/basis/abstract_basis.py b/fedot_ind/core/operation/transformation/basis/abstract_basis.py similarity index 100% rename from fedot_ind/core/operation/implementation/basis/abstract_basis.py rename to fedot_ind/core/operation/transformation/basis/abstract_basis.py diff --git a/fedot_ind/core/operation/transformation/basis/data_driven.py b/fedot_ind/core/operation/transformation/basis/data_driven.py index b65273280..acd5b6f65 100644 --- a/fedot_ind/core/operation/transformation/basis/data_driven.py +++ b/fedot_ind/core/operation/transformation/basis/data_driven.py @@ -1,19 +1,25 @@ -from typing import Optional, TypeVar +from typing import Optional, Tuple, TypeVar +from typing import Optional, Tuple, TypeVar import numpy as np import pandas as pd +import tensorly as tl from fedot.core.data.data import InputData + from fedot.core.operations.operation_parameters import OperationParameters from joblib import Parallel, delayed from pymonad.either import Either from pymonad.list import ListMonad from scipy import stats from scipy.spatial.distance import cdist +from tensorly.decomposition import parafac from tqdm import tqdm -from fedot_ind.core.operation.decomposition.SpectrumDecomposition import SpectrumDecomposer -from fedot_ind.core.operation.implementation.basis.abstract_basis import BasisDecompositionImplementation +from fedot_ind.core.operation.decomposition.matrix_decomposition.fast_svd import RSVDDecomposition +from fedot_ind.core.operation.transformation.basis.abstract_basis import BasisDecompositionImplementation from fedot_ind.core.operation.transformation.data.hankel import HankelMatrix +from fedot_ind.core.operation.transformation.regularization.spectrum import reconstruct_basis, \ + singular_value_hard_threshold class_type = TypeVar("T", bound="DataDrivenBasis") @@ -36,7 +42,8 @@ def __init__(self, params: Optional[OperationParameters] = None): self.SV_threshold = None # self.sv_selector = params.get('sv_selector') self.sv_selector = 'median' - + self.svd_estimator = RSVDDecomposition() + self.low_rank_approximation = True self.logging_params.update({'WS': self.window_size, 'SV_selector': self.sv_selector, }) @@ -89,15 +96,6 @@ def _transform(self, input_data: InputData) -> np.array: # predict = self._clean_predict(np.array(v)) return predict - def _transform_one_sample(self, series: np.array, svd_flag: bool = False): - trajectory_transformer = HankelMatrix(time_series=series, window_size=self.window_size) - data = trajectory_transformer.trajectory_matrix - self.window_size = trajectory_transformer.window_length - self.decomposer = SpectrumDecomposer(data, trajectory_transformer.ts_length, self.SV_threshold) - if svd_flag: - return self.estimate_singular_values(data) - return self._get_basis(data) - def get_threshold(self, data, selector: str): selectors = {'median': stats.mode, @@ -111,20 +109,74 @@ def get_threshold(self, data, selector: str): return selectors[selector](svd_numbers).mode[0] + def _transform_one_sample(self, series: np.array, svd_flag: bool = False): + trajectory_transformer = HankelMatrix(time_series=series, window_size=self.window_size) + data = trajectory_transformer.trajectory_matrix + self.ts_length = trajectory_transformer.ts_length + if svd_flag: + return self.estimate_singular_values(data) + return self._get_basis(data) + def estimate_singular_values(self, data): - basis = Either.insert(data).then(self.decomposer.svd).value[0] + svd = lambda x: ListMonad(self.svd_estimator.rsvd(tensor=x, approximation=self.low_rank_approximation)) + basis = Either.insert(data).then(svd).value[0] spectrum = [s_val for s_val in basis[1] if s_val > 0.001] # self.left_approx_sv, self.right_approx_sv = basis[0], basis[2] return len(spectrum) def _get_1d_basis(self, data): - basis = Either.insert(data).then(self.decomposer.svd).then(self.decomposer.threshold).then( - self.decomposer.data_driven_basis).value[0] + data_driven_basis = lambda Monoid: ListMonad(reconstruct_basis(Monoid[0], + Monoid[1], + Monoid[2], + ts_length=self.ts_length)) + threshold = lambda Monoid: ListMonad([Monoid[0], + Monoid[1][:self.SV_threshold], + Monoid[2]]) + svd = lambda x: ListMonad(self.svd_estimator.rsvd(tensor=x, + approximation=self.low_rank_approximation, + regularized_rank=self.SV_threshold)) + basis = Either.insert(data).then(svd).then(threshold).then(data_driven_basis).value[0] return np.swapaxes(basis, 1, 0) def _get_multidim_basis(self, data): + rank = round(data[0].shape[0] / 10) + beta = data[0].shape[0] / data[0].shape[1] + + tensor_decomposition = lambda x: ListMonad(parafac(tl.tensor(x), rank=rank).factors) + multi_threshold = lambda x: singular_value_hard_threshold(singular_values=x, + beta=beta, + threshold=None) + + threshold = lambda Monoid: ListMonad([Monoid[1], + list(map(multi_threshold, Monoid[0])), + Monoid[2].T]) if self.n_components is None else ListMonad([Monoid[1][ + :, + :self.n_components], + Monoid[0][ + :, + :self.n_components], + Monoid[2][ + :, + :self.n_components].T]) + data_driven_basis = lambda Monoid: ListMonad(reconstruct_basis(Monoid[0], + Monoid[1], + Monoid[2], + ts_length=self.ts_length)) + basis = np.array( - Either.insert(data).then(self.decomposer.tensor_decomposition).then(self.decomposer.multi_threshold).then( - self.decomposer.data_driven_basis).value[0]) + Either.insert(data).then(tensor_decomposition).then(threshold).then(data_driven_basis).value[0]) + basis = basis.reshape(basis.shape[1], -1) + return basis + + def evaluate_derivative(self: class_type, + coefs: np.array, + order: int = 1) -> Tuple[class_type, np.array]: + basis = type(self)( + domain_range=self.domain_range, + n_basis=self.n_basis - order, + ) + derivative_coefs = np.array([np.polyder(x[::-1], order)[::-1] for x in coefs]) + + return basis, derivative_coefs diff --git a/fedot_ind/core/operation/implementation/basis/fourier.py b/fedot_ind/core/operation/transformation/basis/fourier.py similarity index 96% rename from fedot_ind/core/operation/implementation/basis/fourier.py rename to fedot_ind/core/operation/transformation/basis/fourier.py index 359d04a35..4687b7b48 100644 --- a/fedot_ind/core/operation/implementation/basis/fourier.py +++ b/fedot_ind/core/operation/transformation/basis/fourier.py @@ -1,7 +1,7 @@ from typing import Optional import numpy as np from fedot.core.operations.operation_parameters import OperationParameters -from fedot_ind.core.operation.implementation.basis.abstract_basis import BasisDecompositionImplementation +from fedot_ind.core.operation.transformation.basis.abstract_basis import BasisDecompositionImplementation class FourierBasisImplementation(BasisDecompositionImplementation): diff --git a/fedot_ind/core/operation/implementation/basis/wavelet.py b/fedot_ind/core/operation/transformation/basis/wavelet.py similarity index 97% rename from fedot_ind/core/operation/implementation/basis/wavelet.py rename to fedot_ind/core/operation/transformation/basis/wavelet.py index 0d73a1d6c..b9d14cf1c 100644 --- a/fedot_ind/core/operation/implementation/basis/wavelet.py +++ b/fedot_ind/core/operation/transformation/basis/wavelet.py @@ -4,7 +4,7 @@ from fedot.core.operations.operation_parameters import OperationParameters from pymonad.either import Either from pymonad.list import ListMonad -from fedot_ind.core.operation.implementation.basis.abstract_basis import BasisDecompositionImplementation +from fedot_ind.core.operation.transformation.basis.abstract_basis import BasisDecompositionImplementation class WaveletBasisImplementation(BasisDecompositionImplementation): diff --git a/fedot_ind/core/operation/ts_forecasting/__init__.py b/fedot_ind/core/operation/ts_forecasting/__init__.py deleted file mode 100644 index e69de29bb..000000000