moving extractors to new module

fedot_ind/core/models/manifold/riemann_embeding.py → fedot_ind/core/operation/transformation/representation/manifold/riemann_embeding.py

@@ -62,6 +62,9 @@ def __init__(self, params: Optional[OperationParameters] = None):
             'tangent_space_metric': self.tangent_metric,
             'SPD_space_metric': self.spd_metric})
 
+    def __repr__(self):
+        return 'Riemann Manifold Class for TS representation'
+
     def _init_spaces(self):
         self.spd_space = Covariances(estimator='scm')
         self.tangent_space = TangentSpace(metric=self.tangent_metric)

fedot_ind/core/models/recurrence/reccurence_extractor.py → fedot_ind/core/operation/transformation/representation/recurrence/reccurence_extractor.py

@@ -7,9 +7,9 @@
 
 # from fedot_ind.core.metrics.metrics_implementation import *
 from fedot_ind.core.models.base_extractor import BaseExtractor
-from fedot_ind.core.models.recurrence.sequences import RecurrenceFeatureExtractor
 from fedot_ind.core.operation.transformation.data.hankel import HankelMatrix
 from fedot_ind.core.operation.transformation.data.kernel_matrix import TSTransformer
+from fedot_ind.core.operation.transformation.representation.recurrence.sequences import RecurrenceFeatureExtractor
 
 
 class RecurrenceExtractor(BaseExtractor):
@@ -51,6 +51,9 @@ def __init__(self, params: Optional[OperationParameters] = None):
         self.transformer = TSTransformer
         self.extractor = RecurrenceFeatureExtractor
 
+    def __repr__(self):
+        return 'Reccurence Class for TS representation'
+
     def _generate_features_from_ts(self, ts: np.array):
         if self.window_size != 0:
             trajectory_transformer = HankelMatrix(time_series=ts,

fedot_ind/core/models/quantile/quantile_extractor.py → fedot_ind/core/operation/transformation/representation/statistical/quantile_extractor.py

@@ -1,6 +1,6 @@
-from itertools import chain
 from typing import Optional
 
+import dask
 from fedot.core.data.data import InputData
 from fedot.core.operations.operation_parameters import OperationParameters
 
@@ -9,7 +9,7 @@
 
 
 class QuantileExtractor(BaseExtractor):
-    """Class responsible for quantile feature generator experiment.
+    """Class responsible for statistical feature generator experiment.
 
     Attributes:
         window_size (int): size of window
@@ -44,40 +44,30 @@ def __init__(self, params: Optional[OperationParameters] = None):
         self.logging_params.update({'Wsize': self.window_size,
                                     'Stride': self.stride})
 
+    def __repr__(self):
+        return 'Statistical Class for TS representation'
+
     def _concatenate_global_and_local_feature(
             self,
-            global_features: InputData,
-            window_stat_features: InputData) -> InputData:
-
-        if isinstance(window_stat_features.features[0], list):
-            window_stat_features.features = np.concatenate(
-                window_stat_features.features, axis=0)
-            window_stat_features.supplementary_data['feature_name'] = list(
-                chain(*window_stat_features.supplementary_data['feature_name']))
-
-        window_stat_features.features = np.concatenate(
-            [global_features.features, window_stat_features.features], axis=0)
-        window_stat_features.features = np.nan_to_num(
-            window_stat_features.features)
-
-        window_stat_features.supplementary_data['feature_name'] = list(
-            chain(*[global_features.supplementary_data['feature_name'],
-                    window_stat_features.supplementary_data['feature_name']]))
+            global_features: np.ndarray,
+            window_stat_features: np.ndarray) -> np.ndarray:
+
+        if isinstance(window_stat_features[0], list):
+            window_stat_features = np.concatenate(window_stat_features, axis=0)
+
+        window_stat_features = np.concatenate([global_features, window_stat_features], axis=0)
+        window_stat_features = np.nan_to_num(window_stat_features)
         return window_stat_features
 
     def extract_stats_features(self, ts: np.array) -> InputData:
-        global_features = self.get_statistical_features(
-            ts, add_global_features=True)
-        if self.window_size != 0:
-            window_stat_features = self.apply_window_for_stat_feature(
-                ts_data=ts,
-                feature_generator=self.get_statistical_features,
-                window_size=self.window_size)
-        else:
-            window_stat_features = self.get_statistical_features(ts)
+        global_features = self.get_statistical_features(ts, add_global_features=True)
+        window_stat_features = self.get_statistical_features(ts) if self.window_size == 0 else \
+            self.apply_window_for_stat_feature(ts_data=ts, feature_generator=self.get_statistical_features,
+                                               window_size=self.window_size)
         return self._concatenate_global_and_local_feature(
             global_features, window_stat_features) if self.add_global_features else window_stat_features
 
+    @dask.delayed
     def generate_features_from_ts(self,
                                   ts: np.array,
                                   window_length: int = None) -> InputData:

fedot_ind/core/models/quantile/stat_features.py → fedot_ind/core/operation/transformation/representation/statistical/stat_features.py

@@ -1,11 +1,12 @@
 import warnings
 
-from fedot_ind.core.architecture.settings.computational import backend_methods as np
 import pandas as pd
 from scipy.signal import find_peaks
 from scipy.stats import entropy, linregress
 from sklearn.preprocessing import MinMaxScaler
 
+from fedot_ind.core.architecture.settings.computational import backend_methods as np
+
 warnings.filterwarnings("ignore")
 
 
@@ -34,7 +35,7 @@ def diff(array: np.array) -> float:
     return np.diff(array, n=len(array) - 1)[0]
 
 
-# Extra methods for quantile features extraction
+# Extra methods for statistical features extraction
 def skewness(array: np.array) -> float:
     if not isinstance(array, pd.Series):
         array = pd.Series(array)

fedot_ind/core/models/tabular/tabular_extractor.py → fedot_ind/core/operation/transformation/representation/tabular/tabular_extractor.py

@@ -3,16 +3,19 @@
 import numpy as np
 from fedot.core.data.data import InputData
 from fedot.core.operations.operation_parameters import OperationParameters
+from fedot.core.pipelines.pipeline_builder import PipelineBuilder
+from pymonad.either import Either
 from sklearn.decomposition import PCA
 from sklearn.preprocessing import StandardScaler
 
 from fedot_ind.core.models.base_extractor import BaseExtractor
+from fedot_ind.core.operation.transformation.data.park_transformation import park_transform
 from fedot_ind.core.repository.constanst_repository import KERNEL_BASELINE_FEATURE_GENERATORS
 from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels
 
 
 class TabularExtractor(BaseExtractor):
-    """Class responsible for quantile feature generator experiment.
+    """Class responsible for statistical feature generator experiment.
 
     Attributes:
         window_size (int): size of window
@@ -47,6 +50,7 @@ def __init__(self, params: Optional[OperationParameters] = None):
         self.reduce_dimension = params.get('reduce_dimension', True)
 
         self.repo = IndustrialModels().setup_repository()
+        self.custom_tabular_transformation = {'park_transformation': park_transform}
         self.pca_is_fitted = False
         self.scaler = StandardScaler()
         self.pca = PCA(self.explained_dispersion)
@@ -58,6 +62,35 @@ def _reduce_dim(self, features, target):
             self.pca_is_fitted = True
             return self.pca.fit_transform(self.scaler.fit_transform(features, target))
 
+    def _create_from_custom_fg(self, input_data):
+        for model_name, nodes in self.feature_domain.items():
+            if model_name.__contains__('custom'):
+                transform_method = self.custom_tabular_transformation[nodes[0]]
+                ts_representation = transform_method(input_data)
+            else:
+                model = PipelineBuilder()
+                for node in nodes:
+                    if isinstance(node, tuple):
+                        model.add_node(operation_type=node[0], params=node[1])
+                    else:
+                        model.add_node(operation_type=node)
+                model = model.build()
+                ts_representation = model.fit(input_data).predict
+            self.feature_list.append(ts_representation)
+
+    def _create_from_default_fg(self, input_data):
+        feature_domain_models = [model for model in KERNEL_BASELINE_FEATURE_GENERATORS]
+
+        if not self.feature_domain.__contains__('all'):
+            feature_domain_models = [model for model in feature_domain_models
+                                     if model.__contains__(self.feature_domain)]
+
+        for model_name in feature_domain_models:
+            model = KERNEL_BASELINE_FEATURE_GENERATORS[model_name]
+            model.heads[0].parameters['use_sliding_window'] = self.use_sliding_window
+            model = model.build()
+            self.feature_list.append(model.fit(input_data).predict)
+
     def create_feature_matrix(self, feature_list: list):
         return np.concatenate([x.reshape(x.shape[0], x.shape[1] * x.shape[2])
                                for x in feature_list], axis=1).squeeze()
@@ -74,17 +107,10 @@ def _transform(self, input_data: InputData) -> np.array:
     def generate_features_from_ts(self,
                                   input_data: InputData,
                                   window_length: int = None) -> InputData:
-        feature_domain_models = [model for model in KERNEL_BASELINE_FEATURE_GENERATORS]
+        is_custom_feature_representation = isinstance(self.feature_domain, dict)
         self.feature_list = []
-
-        if not self.feature_domain.__contains__('all'):
-            feature_domain_models = [model for model in feature_domain_models
-                                     if model.__contains__(self.feature_domain)]
-
-        for model_name in feature_domain_models:
-            model = KERNEL_BASELINE_FEATURE_GENERATORS[model_name]
-            model.heads[0].parameters['use_sliding_window'] = self.use_sliding_window
-            model = model.build()
-            self.feature_list.append(model.fit(input_data).predict)
-
+        Either(value=input_data,
+               monoid=[input_data,
+                       is_custom_feature_representation]).either(left_function=self._create_from_default_fg,
+                                                                 right_function=self._create_from_custom_fg)
         return self.feature_list

fedot_ind/core/models/topological/topological_extractor.py → fedot_ind/core/operation/transformation/representation/topological/topological_extractor.py

@@ -1,20 +1,26 @@
 import sys
 from functools import partial
+from itertools import product
+from typing import Optional
 
+import open3d as o3d
 import pandas as pd
 from fedot.core.data.data import InputData
 from fedot.core.operations.operation_parameters import OperationParameters
 from fedot.core.repository.dataset_types import DataTypesEnum
+from gtda.homology import VietorisRipsPersistence
 from gtda.time_series import takens_embedding_optimal_parameters
 from scipy import stats
+from scipy.spatial.distance import squareform, pdist
 from tqdm import tqdm
-from typing import Optional
 
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
 from fedot_ind.core.models.base_extractor import BaseExtractor
-from fedot_ind.core.models.topological.topofeatures import PersistenceDiagramsExtractor, TopologicalFeaturesExtractor
 from fedot_ind.core.operation.transformation.data.point_cloud import TopologicalTransformation
+from fedot_ind.core.operation.transformation.representation.topological.topofeatures import \
+    PersistenceDiagramsExtractor, TopologicalFeaturesExtractor
 from fedot_ind.core.repository.constanst_repository import PERSISTENCE_DIAGRAM_EXTRACTOR, PERSISTENCE_DIAGRAM_FEATURES
+from fedot_ind.tools.explain.pcd import numpy2stl
 
 sys.setrecursionlimit(1000000000)
 
@@ -53,6 +59,10 @@ def __init__(self, params: Optional[OperationParameters] = None):
             persistence_diagram_features=PERSISTENCE_DIAGRAM_FEATURES
         )
         self.data_transformer = None
+        self.save_pcd = False
+
+    def __repr__(self):
+        return 'Topological Class for TS representation'
 
     def __evaluate_persistence_params(self, ts_data: np.array):
         if self.feature_extractor is None:
@@ -67,12 +77,43 @@ def __evaluate_persistence_params(self, ts_data: np.array):
                 persistence_diagram_extractor=persistence_diagram_extractor,
                 persistence_diagram_features=PERSISTENCE_DIAGRAM_FEATURES)
 
+    def _generate_vr_mesh(self, pcd):
+        # Corresponding matrix of Euclidean pairwise distances
+        pairwise_distances = squareform(pdist(pcd))
+        # Default parameter for ``metric`` is "euclidean"
+        vr_graph = VietorisRipsPersistence(metric="precomputed").fit_transform([pairwise_distances])
+        return vr_graph
+
+    def _generate_pcd(self, ts_data, persistence_params):
+        window_size_range = list(range(1, 35, 5))
+        stride_range = list(range(1, 15, 3))
+        pcd_params = list(product(window_size_range, stride_range))
+        for params in pcd_params:
+            data_transformer = TopologicalTransformation(stride=params[1], persistence_params=persistence_params,
+                                                         window_length=round(ts_data.shape[0] * 0.01 * params[0]))
+            point_cloud = data_transformer.time_series_to_point_cloud(input_data=ts_data, use_gtda=True)
+            # VR_mesh = self._generate_vr_mesh(point_cloud)
+            for scale in range(1, 15, 3):
+                numpy2stl(point_cloud,
+                          f"./stl_scale_{scale}_ws_{params[0]}_stride_{params[1]}.stl",
+                          max_width=300.,
+                          max_depth=200.,
+                          max_height=300.,
+                          scale=scale,
+                          min_thickness_percent=0.5,
+                          solid=False)
+            pcd = o3d.geometry.PointCloud()
+            pcd.points = o3d.utility.Vector3dVector(point_cloud)
+            o3d.io.write_point_cloud(f"./pcd_ws_{params[0]}_stride_{params[1]}.ply", pcd)
+
     def _generate_features_from_ts(self, ts_data: np.array, persistence_params: dict) -> InputData:
+        if self.save_pcd:
+            self._generate_pcd(ts_data, persistence_params)
         if self.data_transformer is None:
             self.data_transformer = TopologicalTransformation(
                 persistence_params=persistence_params, window_length=round(ts_data.shape[0] * 0.01 * self.window_size))
 
-        point_cloud = self.data_transformer.time_series_to_point_cloud(input_data=ts_data)
+        point_cloud = self.data_transformer.time_series_to_point_cloud(input_data=ts_data, use_gtda=True)
         topological_features = self.feature_extractor.transform(point_cloud)
         topological_features = InputData(idx=np.arange(len(topological_features.values)),
                                          features=topological_features.values,

tests/unit/core/models/test_quantile_extractor.py

@@ -4,7 +4,7 @@
 
 from fedot_ind.api.utils.data import init_input_data
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
-from fedot_ind.core.models.quantile.quantile_extractor import QuantileExtractor
+from fedot_ind.core.operation.transformation.representation.statistical.quantile_extractor import QuantileExtractor
 from fedot_ind.core.repository.constanst_repository import STAT_METHODS, STAT_METHODS_GLOBAL
 from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator
 

tests/unit/core/models/test_recurrence_extractor.py

@@ -4,7 +4,7 @@
 
 from fedot_ind.api.utils.data import init_input_data
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
-from fedot_ind.core.models.recurrence.reccurence_extractor import RecurrenceExtractor
+from fedot_ind.core.operation.transformation.representation.recurrence import RecurrenceExtractor
 from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator
 
 

tests/unit/core/models/test_riemann_embeding.py

@@ -6,7 +6,7 @@
 
 from fedot_ind.api.utils.data import init_input_data
 from fedot_ind.api.utils.path_lib import PATH_TO_DEFAULT_PARAMS
-from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor
+from fedot_ind.core.operation.transformation.representation.manifold.riemann_embeding import RiemannExtractor
 from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator
 
 

tests/unit/core/models/test_topological_extractor.py

@@ -1,9 +1,9 @@
 import pytest
 from fedot.core.data.data import InputData, OutputData
-from fedot_ind.api.utils.data import init_input_data
 
+from fedot_ind.api.utils.data import init_input_data
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
-from fedot_ind.core.models.topological.topological_extractor import TopologicalExtractor
+from fedot_ind.core.operation.transformation.representation.topological import TopologicalExtractor
 from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator