add dask as backend for models, refactoring in constant and model repo

examples/tutorial/time_series/ts_classification/tmp.py

@@ -78,6 +78,7 @@ def plot_mean_sample_multi(X, y, labels: list = [], n_channel: int = None):
                   metric='accuracy',
                   timeout=15,
                   pop_size=20,
+                  backend='dask',
                   with_tunig=False,
                   n_jobs=-1,
                   logging_level=20)

fedot_ind/api/main.py

@@ -74,7 +74,7 @@ def __init_solver(self):
             self.repo = IndustrialModels().setup_default_repository()
             self.config_dict['optimizer'] = None
         else:
-            self.repo = IndustrialModels().setup_repository()
+            self.repo = IndustrialModels().setup_repository(backend=self.api_controller.backend_method)
         self.logger.info(f'-------------------------------------------------')
         self.logger.info('Initialising Dask Server')
         self.config_dict['initial_assumption'] = self.config_dict['initial_assumption'].build()
@@ -83,8 +83,6 @@ def __init_solver(self):
         self.logger.info(f'-------------------------------------------------')
         self.logger.info('Initialising solver')
         self.solver = Fedot(**self.config_dict)
-        # if self.api_controller.is_default_fedot_context:
-        #     self.solver = self.api_controller._check_mutations(self.solver)
 
     def _process_input_data(self, input_data):
         train_data = deepcopy(input_data)  # we do not want to make inplace changes

fedot_ind/api/utils/api_init.py

@@ -2,16 +2,14 @@
 from pathlib import Path
 
 from fedot.core.repository.tasks import TsForecastingParams
-from golem.core.optimisers.adaptive.operator_agent import RandomAgent
 from pymonad.either import Either
 
 from fedot_ind.api.utils.industrial_strategy import IndustrialStrategy
 from fedot_ind.api.utils.path_lib import DEFAULT_PATH_RESULTS as default_path_to_save_results
 from fedot_ind.core.architecture.preprocessing.data_convertor import ApiConverter
-from fedot_ind.core.architecture.settings.computational import BackendMethods
 from fedot_ind.core.optimizer.IndustrialEvoOptimizer import IndustrialEvoOptimizer
 from fedot_ind.core.repository.constanst_repository import \
-    FEDOT_API_PARAMS, fedot_init_assumptions, FEDOT_MUTATION_STRATEGY
+    FEDOT_API_PARAMS, fedot_init_assumptions
 from fedot_ind.core.repository.model_repository import default_industrial_availiable_operation
 from fedot_ind.tools.explain.explain import PointExplainer, RecurrenceExplainer
 
@@ -118,26 +116,11 @@ def __init_experiment_setup(self):
         self.logger.info('Initialising experiment setup')
 
         industrial_params = set(self.config_dict.keys()) - \
-            set(FEDOT_API_PARAMS.keys())
+                            set(FEDOT_API_PARAMS.keys())
         for param in industrial_params:
             self.config_dict.pop(param, None)
 
-        backend_method_current, backend_scipy_current = BackendMethods(
-            self.backend_method).backend
-        globals()['backend_methods'] = backend_method_current
-        globals()['backend_scipy'] = backend_scipy_current
-
-    def _check_mutations(self, solver):
-        for mutation in solver.api_composer.params.optimizer_params.mutation_types.mutation_types:
-            try:
-                is_invalid = mutation.__name__.__contains__('resample')
-            except Exception:
-                is_invalid = mutation.name.__contains__('resample')
-            if is_invalid:
-                solver.api_composer.params.optimizer_params.mutation_types.mutation_types.remove(mutation)
-
-        solver.api_composer.params.optimizer_params.adaptive_mutation_type = RandomAgent(
-            actions=solver.api_composer.params.optimizer_params.mutation_types,
-            probs=FEDOT_MUTATION_STRATEGY[
-                'params_mutation_strategy'])
-        return solver
+        # backend_method_current, backend_scipy_current = BackendMethods(
+        #     self.backend_method).backend
+        # globals()['backend_methods'] = backend_method_current
+        # globals()['backend_scipy'] = backend_scipy_current

fedot_ind/core/operation/decomposition/matrix_decomposition/column_sampling_decomposition.py

@@ -5,6 +5,7 @@
 from sklearn.random_projection import johnson_lindenstrauss_min_dim
 
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
+from fedot_ind.core.repository.model_repository import DEFAULT_SVD_SOLVER
 
 
 class CURDecomposition:
@@ -59,7 +60,7 @@ def fit_transform(self, feature_tensor: np.ndarray,
             sampled_tensor = sampled_tensor[self.row_indices, :]
         else:
             # evaluate pseudoinverse for W - U^-1
-            X, Sigma, y_T = np.linalg.svd(w, full_matrices=False)
+            X, Sigma, y_T = DEFAULT_SVD_SOLVER(w, full_matrices=False)
             Sigma_plus = np.linalg.pinv(np.diag(Sigma))
             # aprox U using pseudoinverse
             u = y_T.T @ Sigma_plus @ Sigma_plus @ X.T
@@ -135,46 +136,3 @@ def matrix_to_ts(matrix: np.ndarray) -> np.ndarray:
             ts[i:i + matrix.shape[1]] += matrix[i]
         return ts
 
-
-def get_random_sparse_matrix(size: tuple):
-    """Generate random sparse matrix with size = size"""
-
-    matrix = np.zeros(size)
-    for i in range(size[0]):
-        for j in range(size[1]):
-            if np.random.rand() < 0.1:
-                matrix[i, j] = np.random.rand()
-    return matrix
-
-
-if __name__ == '__main__':
-    from fedot_ind.tools.loader import DataLoader
-
-    arr = np.array([[1, 1, 1, 0, 0],
-                    [3, 3, 3, 0, 0],
-                    [4, 4, 4, 0, 0],
-                    [5, 5, 5, 0, 0],
-                    [0, 0, 0, 4, 4],
-                    [0, 0, 0, 5, 5],
-                    [0, 0, 0, 2, 2]])
-
-    (X_train, y_train), (X_test, y_test) = DataLoader('Lightning7').load_data()
-
-    # init_ts = train[0].iloc[0, :].values
-    # scaler = MinMaxScaler()
-    # scaler.fit(init_ts.reshape(-1, 1))
-    # single_ts = scaler.transform(init_ts.reshape(-1, 1)).reshape(-1)
-
-    cur = CURDecomposition(rank=20)
-    # M = cur.ts_to_matrix(single_ts, 30)
-    C, U, R = cur.fit_transform(X_train)
-    basis = cur.reconstruct_basis(C, U, R, X_train.shape[1])
-
-    # rec_ts = cur.matrix_to_ts(C @ U @ R)
-    # err = np.linalg.norm(single_ts - rec_ts)
-
-    # plt.plot(init_ts, label='init_ts')
-    # plt.plot(scaler.inverse_transform(rec_ts.reshape(-1, 1)), label='rec_ts')
-    # plt.legend()
-    # plt.show()
-    _ = 1

fedot_ind/core/operation/decomposition/matrix_decomposition/dmd_decomposition.py

@@ -1,10 +1,9 @@
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
-from numpy.linalg import svd
-
+from fedot_ind.core.repository.model_repository import DEFAULT_SVD_SOLVER, DEFAULT_QR_SOLVER
 
 def rq(A):
     n, m = A.shape
-    Q, R = np.linalg.qr(np.flipud(A).T, mode='complete')
+    Q, R = DEFAULT_QR_SOLVER(np.flipud(A).T, mode='complete')
     R = np.rot90(R.T, 2)
     Q = np.flipud(Q.T)
     if n > m:
@@ -18,7 +17,7 @@ def tls(A, B):
     if A.shape[0] != B.shape[0]:
         raise ValueError('Matrices are not conformant.')
     R1 = np.hstack((A, B))
-    U, S, V = np.linalg.svd(R1)
+    U, S, V = DEFAULT_SVD_SOLVER(R1)
     r = B.shape[1]
     R, Q = rq(V[:, r:])
     Gamma = R[n:, n - r:]
@@ -28,7 +27,7 @@ def tls(A, B):
 
 
 def exact_dmd_decompose(X, Y, rank):
-    Ux, Sx, Vx = svd(X)
+    Ux, Sx, Vx = DEFAULT_SVD_SOLVER(X)
     Ux = Ux[:, :rank]
     Sx = Sx[:rank]
     Sx = np.diag(Sx)
@@ -46,14 +45,14 @@ def A(v): return np.dot(a=Ux, b=np.dot(a=Atilde, b=np.dot(a=Ux.T, b=v)))
 
 
 def orthogonal_dmd_decompose(X, Y, rank):
-    Ux, _, _ = svd(X)
+    Ux, _, _ = DEFAULT_SVD_SOLVER(X)
     Ux = Ux[:, :rank]
     # Project X (current state) and Y (future state) on leading components of X
     Yproj = Ux.T @ Y
     Xproj = Ux.T @ X
     # A_proj is constrained to be a unitary matrix and the minimization problem is argmin (A.T @ A = I) |Y-AX|_frob
     # The solution of A_proj is obtained by Schonemann A = Uyx,@ Vyx.T
-    Uyx, _, Vyx = svd(Yproj @ Xproj.T)
+    Uyx, _, Vyx = DEFAULT_SVD_SOLVER(Yproj @ Xproj.T)
     Aproj = Uyx @ Vyx.T
     def A(x): return np.dot(a=Ux, b=np.dot(a=Aproj, b=np.dot(a=Ux.T, b=x)))
     # Diagonalise unitary operator
@@ -65,7 +64,7 @@ def A(x): return np.dot(a=Ux, b=np.dot(a=Aproj, b=np.dot(a=Ux.T, b=x)))
 
 
 def symmetric_decompose(X, Y, rank):
-    Ux, S, V = np.linalg.svd(X)
+    Ux, S, V = DEFAULT_SVD_SOLVER(X)
     C = np.dot(Ux.T, np.dot(Y, V))
     C1 = C
     if rank is None:

fedot_ind/core/operation/decomposition/matrix_decomposition/power_iteration_decomposition.py

@@ -7,6 +7,7 @@
 from fedot_ind.core.operation.filtration.channel_filtration import _detect_knee_point
 from fedot_ind.core.operation.transformation.regularization.spectrum import singular_value_hard_threshold, \
     sv_to_explained_variance_ratio, eigencorr_matrix
+from fedot_ind.core.repository.model_repository import DEFAULT_SVD_SOLVER, DEFAULT_QR_SOLVER
 
 
 class RSVDDecomposition:
@@ -81,7 +82,7 @@ def rsvd(self,
         # thresholding
         if not approximation:
             # classic svd decomposition
-            Ut, St, Vt = np.linalg.svd(tensor, full_matrices=False)
+            Ut, St, Vt = DEFAULT_SVD_SOLVER(tensor, full_matrices=False)
             # Compute low rank.
             low_rank = self._spectrum_regularization(St, reg_type=reg_type)
             if regularized_rank is not None:
@@ -110,14 +111,13 @@ def rsvd(self,
                 AAT, self.poly_deg) @ tensor @ self.random_projection
             # Fourth step. Orthogonalization of the resulting "sampled" matrix
             # creates for us a basis of eigenvectors.
-            sampled_tensor_orto, _ = np.linalg.qr(
-                sampled_tensor, mode='reduced')
+            sampled_tensor_orto, _ = DEFAULT_QR_SOLVER(sampled_tensor, mode='reduced')
             # Fifth step. Project initial Gramm matrix on new basis obtained
             # from "sampled matrix".
             M = sampled_tensor_orto.T @ AAT @ sampled_tensor_orto
             # Six step. Classical svd decomposition with choosen type of
             # spectrum thresholding
-            Ut, St, Vt = np.linalg.svd(M, full_matrices=False)
+            Ut, St, Vt = DEFAULT_SVD_SOLVER(M, full_matrices=False)
             # Compute low rank.
             low_rank = self._spectrum_regularization(St, reg_type=reg_type)
             # Seven step. Compute matrix approximation and choose new low_rank
@@ -127,6 +127,6 @@ def rsvd(self,
             # Eight step. Return matrix approximation.
             reconstr_tensor = self._compute_matrix_approximation(
                 Ut, sampled_tensor_orto, tensor, regularized_rank)
-            U_, S_, V_ = np.linalg.svd(reconstr_tensor, full_matrices=False)
+            U_, S_, V_ = DEFAULT_SVD_SOLVER(reconstr_tensor, full_matrices=False)
 
             return [U_, S_, V_]

fedot_ind/core/repository/constanst_repository.py

@@ -1,4 +1,5 @@
 import math
+import pathlib
 from enum import Enum
 from multiprocessing import cpu_count
 
@@ -16,6 +17,7 @@
     minkowski
 from torch import nn
 
+from fedot_ind.api.utils.path_lib import PROJECT_PATH
 from fedot_ind.core.metrics.metrics_implementation import calculate_classification_metric, calculate_regression_metric, \
     calculate_forecasting_metric, calculate_detection_metric
 from fedot_ind.core.models.nn.network_modules.losses import CenterLoss, CenterPlusLoss, ExpWeightedLoss, FocalLoss, \
@@ -62,7 +64,7 @@
 
 def beta_thr(beta):
     return 0.56 * np.power(beta, 3) - 0.95 * \
-        np.power(beta, 2) + 1.82 * beta + 1.43
+           np.power(beta, 2) + 1.82 * beta + 1.43
 
 
 def get_default_industrial_model_params(model_name):
@@ -147,6 +149,15 @@ class DataTypeConstant(Enum):
     TRAJECTORY_MATRIX = HankelMatrix
 
 
+class PathConstant(Enum):
+    IND_DATA_OPERATION_PATH = pathlib.Path(PROJECT_PATH, 'fedot_ind', 'core', 'repository', 'data',
+                                           'industrial_data_operation_repository.json')
+    DEFAULT_DATA_OPERATION_PATH = pathlib.Path('data_operation_repository.json')
+    IND_MODEL_OPERATION_PATH = pathlib.Path(PROJECT_PATH, 'fedot_ind', 'core', 'repository', 'data',
+                                            'industrial_model_repository.json')
+    DEFAULT_MODEL_OPERATION_PATH = pathlib.Path('model_repository.json')
+
+
 class FeatureConstant(Enum):
     STAT_METHODS = {
         'mean_': np.mean,
@@ -781,6 +792,11 @@ class UnitTestConstant(Enum):
 MATRIX = DataTypeConstant.MATRIX.value
 TRAJECTORY_MATRIX = DataTypeConstant.TRAJECTORY_MATRIX.value
 
+IND_MODEL_OPERATION_PATH = PathConstant.IND_MODEL_OPERATION_PATH.value
+IND_DATA_OPERATION_PATH = PathConstant.IND_DATA_OPERATION_PATH.value
+DEFAULT_DATA_OPERATION_PATH = PathConstant.DEFAULT_DATA_OPERATION_PATH.value
+DEFAULT_MODEL_OPERATION_PATH = PathConstant.DEFAULT_MODEL_OPERATION_PATH.value
+
 ENERGY_THR = ModelCompressionConstant.ENERGY_THR.value
 DECOMPOSE_MODE = ModelCompressionConstant.DECOMPOSE_MODE.value
 FORWARD_MODE = ModelCompressionConstant.FORWARD_MODE.value