diff --git a/fedot_ind/core/models/quantile/__init__.py b/fedot_ind/core/operation/transformation/representation/__init__.py similarity index 100% rename from fedot_ind/core/models/quantile/__init__.py rename to fedot_ind/core/operation/transformation/representation/__init__.py diff --git a/fedot_ind/core/models/recurrence/__init__.py b/fedot_ind/core/operation/transformation/representation/manifold/__init__.py similarity index 100% rename from fedot_ind/core/models/recurrence/__init__.py rename to fedot_ind/core/operation/transformation/representation/manifold/__init__.py diff --git a/fedot_ind/core/models/manifold/riemann_embeding.py b/fedot_ind/core/operation/transformation/representation/manifold/riemann_embeding.py similarity index 98% rename from fedot_ind/core/models/manifold/riemann_embeding.py rename to fedot_ind/core/operation/transformation/representation/manifold/riemann_embeding.py index 5fd918a7e..7e6487015 100644 --- a/fedot_ind/core/models/manifold/riemann_embeding.py +++ b/fedot_ind/core/operation/transformation/representation/manifold/riemann_embeding.py @@ -62,6 +62,9 @@ def __init__(self, params: Optional[OperationParameters] = None): 'tangent_space_metric': self.tangent_metric, 'SPD_space_metric': self.spd_metric}) + def __repr__(self): + return 'Riemann Manifold Class for TS representation' + def _init_spaces(self): self.spd_space = Covariances(estimator='scm') self.tangent_space = TangentSpace(metric=self.tangent_metric) diff --git a/fedot_ind/core/models/tabular/__init__.py b/fedot_ind/core/operation/transformation/representation/recurrence/__init__.py similarity index 100% rename from fedot_ind/core/models/tabular/__init__.py rename to fedot_ind/core/operation/transformation/representation/recurrence/__init__.py diff --git a/fedot_ind/core/models/recurrence/reccurence_extractor.py b/fedot_ind/core/operation/transformation/representation/recurrence/reccurence_extractor.py similarity index 95% rename from fedot_ind/core/models/recurrence/reccurence_extractor.py rename to fedot_ind/core/operation/transformation/representation/recurrence/reccurence_extractor.py index f59b05a66..e5cbb80b9 100644 --- a/fedot_ind/core/models/recurrence/reccurence_extractor.py +++ b/fedot_ind/core/operation/transformation/representation/recurrence/reccurence_extractor.py @@ -7,9 +7,9 @@ # from fedot_ind.core.metrics.metrics_implementation import * from fedot_ind.core.models.base_extractor import BaseExtractor -from fedot_ind.core.models.recurrence.sequences import RecurrenceFeatureExtractor from fedot_ind.core.operation.transformation.data.hankel import HankelMatrix from fedot_ind.core.operation.transformation.data.kernel_matrix import TSTransformer +from fedot_ind.core.operation.transformation.representation.recurrence.sequences import RecurrenceFeatureExtractor class RecurrenceExtractor(BaseExtractor): @@ -51,6 +51,9 @@ def __init__(self, params: Optional[OperationParameters] = None): self.transformer = TSTransformer self.extractor = RecurrenceFeatureExtractor + def __repr__(self): + return 'Reccurence Class for TS representation' + def _generate_features_from_ts(self, ts: np.array): if self.window_size != 0: trajectory_transformer = HankelMatrix(time_series=ts, diff --git a/fedot_ind/core/models/topological/__init__.py b/fedot_ind/core/operation/transformation/representation/statistical/__init__.py similarity index 100% rename from fedot_ind/core/models/topological/__init__.py rename to fedot_ind/core/operation/transformation/representation/statistical/__init__.py diff --git a/fedot_ind/core/models/quantile/quantile_extractor.py b/fedot_ind/core/operation/transformation/representation/statistical/quantile_extractor.py similarity index 64% rename from fedot_ind/core/models/quantile/quantile_extractor.py rename to fedot_ind/core/operation/transformation/representation/statistical/quantile_extractor.py index 17a44e0cd..6c19391a5 100644 --- a/fedot_ind/core/models/quantile/quantile_extractor.py +++ b/fedot_ind/core/operation/transformation/representation/statistical/quantile_extractor.py @@ -1,6 +1,6 @@ -from itertools import chain from typing import Optional +import dask from fedot.core.data.data import InputData from fedot.core.operations.operation_parameters import OperationParameters @@ -9,7 +9,7 @@ class QuantileExtractor(BaseExtractor): - """Class responsible for quantile feature generator experiment. + """Class responsible for statistical feature generator experiment. Attributes: window_size (int): size of window @@ -44,40 +44,30 @@ def __init__(self, params: Optional[OperationParameters] = None): self.logging_params.update({'Wsize': self.window_size, 'Stride': self.stride}) + def __repr__(self): + return 'Statistical Class for TS representation' + def _concatenate_global_and_local_feature( self, - global_features: InputData, - window_stat_features: InputData) -> InputData: - - if isinstance(window_stat_features.features[0], list): - window_stat_features.features = np.concatenate( - window_stat_features.features, axis=0) - window_stat_features.supplementary_data['feature_name'] = list( - chain(*window_stat_features.supplementary_data['feature_name'])) - - window_stat_features.features = np.concatenate( - [global_features.features, window_stat_features.features], axis=0) - window_stat_features.features = np.nan_to_num( - window_stat_features.features) - - window_stat_features.supplementary_data['feature_name'] = list( - chain(*[global_features.supplementary_data['feature_name'], - window_stat_features.supplementary_data['feature_name']])) + global_features: np.ndarray, + window_stat_features: np.ndarray) -> np.ndarray: + + if isinstance(window_stat_features[0], list): + window_stat_features = np.concatenate(window_stat_features, axis=0) + + window_stat_features = np.concatenate([global_features, window_stat_features], axis=0) + window_stat_features = np.nan_to_num(window_stat_features) return window_stat_features def extract_stats_features(self, ts: np.array) -> InputData: - global_features = self.get_statistical_features( - ts, add_global_features=True) - if self.window_size != 0: - window_stat_features = self.apply_window_for_stat_feature( - ts_data=ts, - feature_generator=self.get_statistical_features, - window_size=self.window_size) - else: - window_stat_features = self.get_statistical_features(ts) + global_features = self.get_statistical_features(ts, add_global_features=True) + window_stat_features = self.get_statistical_features(ts) if self.window_size == 0 else \ + self.apply_window_for_stat_feature(ts_data=ts, feature_generator=self.get_statistical_features, + window_size=self.window_size) return self._concatenate_global_and_local_feature( global_features, window_stat_features) if self.add_global_features else window_stat_features + @dask.delayed def generate_features_from_ts(self, ts: np.array, window_length: int = None) -> InputData: diff --git a/fedot_ind/core/models/quantile/stat_features.py b/fedot_ind/core/operation/transformation/representation/statistical/stat_features.py similarity index 99% rename from fedot_ind/core/models/quantile/stat_features.py rename to fedot_ind/core/operation/transformation/representation/statistical/stat_features.py index b00e9d811..8e087da14 100644 --- a/fedot_ind/core/models/quantile/stat_features.py +++ b/fedot_ind/core/operation/transformation/representation/statistical/stat_features.py @@ -1,11 +1,12 @@ import warnings -from fedot_ind.core.architecture.settings.computational import backend_methods as np import pandas as pd from scipy.signal import find_peaks from scipy.stats import entropy, linregress from sklearn.preprocessing import MinMaxScaler +from fedot_ind.core.architecture.settings.computational import backend_methods as np + warnings.filterwarnings("ignore") @@ -34,7 +35,7 @@ def diff(array: np.array) -> float: return np.diff(array, n=len(array) - 1)[0] -# Extra methods for quantile features extraction +# Extra methods for statistical features extraction def skewness(array: np.array) -> float: if not isinstance(array, pd.Series): array = pd.Series(array) diff --git a/fedot_ind/core/operation/transformation/representation/tabular/__init__.py b/fedot_ind/core/operation/transformation/representation/tabular/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fedot_ind/core/models/tabular/tabular_extractor.py b/fedot_ind/core/operation/transformation/representation/tabular/tabular_extractor.py similarity index 71% rename from fedot_ind/core/models/tabular/tabular_extractor.py rename to fedot_ind/core/operation/transformation/representation/tabular/tabular_extractor.py index bb00f650f..c50f8bcad 100644 --- a/fedot_ind/core/models/tabular/tabular_extractor.py +++ b/fedot_ind/core/operation/transformation/representation/tabular/tabular_extractor.py @@ -3,16 +3,19 @@ import numpy as np from fedot.core.data.data import InputData from fedot.core.operations.operation_parameters import OperationParameters +from fedot.core.pipelines.pipeline_builder import PipelineBuilder +from pymonad.either import Either from sklearn.decomposition import PCA from sklearn.preprocessing import StandardScaler from fedot_ind.core.models.base_extractor import BaseExtractor +from fedot_ind.core.operation.transformation.data.park_transformation import park_transform from fedot_ind.core.repository.constanst_repository import KERNEL_BASELINE_FEATURE_GENERATORS from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels class TabularExtractor(BaseExtractor): - """Class responsible for quantile feature generator experiment. + """Class responsible for statistical feature generator experiment. Attributes: window_size (int): size of window @@ -47,6 +50,7 @@ def __init__(self, params: Optional[OperationParameters] = None): self.reduce_dimension = params.get('reduce_dimension', True) self.repo = IndustrialModels().setup_repository() + self.custom_tabular_transformation = {'park_transformation': park_transform} self.pca_is_fitted = False self.scaler = StandardScaler() self.pca = PCA(self.explained_dispersion) @@ -58,6 +62,35 @@ def _reduce_dim(self, features, target): self.pca_is_fitted = True return self.pca.fit_transform(self.scaler.fit_transform(features, target)) + def _create_from_custom_fg(self, input_data): + for model_name, nodes in self.feature_domain.items(): + if model_name.__contains__('custom'): + transform_method = self.custom_tabular_transformation[nodes[0]] + ts_representation = transform_method(input_data) + else: + model = PipelineBuilder() + for node in nodes: + if isinstance(node, tuple): + model.add_node(operation_type=node[0], params=node[1]) + else: + model.add_node(operation_type=node) + model = model.build() + ts_representation = model.fit(input_data).predict + self.feature_list.append(ts_representation) + + def _create_from_default_fg(self, input_data): + feature_domain_models = [model for model in KERNEL_BASELINE_FEATURE_GENERATORS] + + if not self.feature_domain.__contains__('all'): + feature_domain_models = [model for model in feature_domain_models + if model.__contains__(self.feature_domain)] + + for model_name in feature_domain_models: + model = KERNEL_BASELINE_FEATURE_GENERATORS[model_name] + model.heads[0].parameters['use_sliding_window'] = self.use_sliding_window + model = model.build() + self.feature_list.append(model.fit(input_data).predict) + def create_feature_matrix(self, feature_list: list): return np.concatenate([x.reshape(x.shape[0], x.shape[1] * x.shape[2]) for x in feature_list], axis=1).squeeze() @@ -74,17 +107,10 @@ def _transform(self, input_data: InputData) -> np.array: def generate_features_from_ts(self, input_data: InputData, window_length: int = None) -> InputData: - feature_domain_models = [model for model in KERNEL_BASELINE_FEATURE_GENERATORS] + is_custom_feature_representation = isinstance(self.feature_domain, dict) self.feature_list = [] - - if not self.feature_domain.__contains__('all'): - feature_domain_models = [model for model in feature_domain_models - if model.__contains__(self.feature_domain)] - - for model_name in feature_domain_models: - model = KERNEL_BASELINE_FEATURE_GENERATORS[model_name] - model.heads[0].parameters['use_sliding_window'] = self.use_sliding_window - model = model.build() - self.feature_list.append(model.fit(input_data).predict) - + Either(value=input_data, + monoid=[input_data, + is_custom_feature_representation]).either(left_function=self._create_from_default_fg, + right_function=self._create_from_custom_fg) return self.feature_list diff --git a/fedot_ind/core/operation/transformation/representation/topological/__init__.py b/fedot_ind/core/operation/transformation/representation/topological/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/fedot_ind/core/models/topological/topological_extractor.py b/fedot_ind/core/operation/transformation/representation/topological/topological_extractor.py similarity index 73% rename from fedot_ind/core/models/topological/topological_extractor.py rename to fedot_ind/core/operation/transformation/representation/topological/topological_extractor.py index ac512d428..2a25cf718 100644 --- a/fedot_ind/core/models/topological/topological_extractor.py +++ b/fedot_ind/core/operation/transformation/representation/topological/topological_extractor.py @@ -1,20 +1,26 @@ import sys from functools import partial +from itertools import product +from typing import Optional +import open3d as o3d import pandas as pd from fedot.core.data.data import InputData from fedot.core.operations.operation_parameters import OperationParameters from fedot.core.repository.dataset_types import DataTypesEnum +from gtda.homology import VietorisRipsPersistence from gtda.time_series import takens_embedding_optimal_parameters from scipy import stats +from scipy.spatial.distance import squareform, pdist from tqdm import tqdm -from typing import Optional from fedot_ind.core.architecture.settings.computational import backend_methods as np from fedot_ind.core.models.base_extractor import BaseExtractor -from fedot_ind.core.models.topological.topofeatures import PersistenceDiagramsExtractor, TopologicalFeaturesExtractor from fedot_ind.core.operation.transformation.data.point_cloud import TopologicalTransformation +from fedot_ind.core.operation.transformation.representation.topological.topofeatures import \ + PersistenceDiagramsExtractor, TopologicalFeaturesExtractor from fedot_ind.core.repository.constanst_repository import PERSISTENCE_DIAGRAM_EXTRACTOR, PERSISTENCE_DIAGRAM_FEATURES +from fedot_ind.tools.explain.pcd import numpy2stl sys.setrecursionlimit(1000000000) @@ -53,6 +59,10 @@ def __init__(self, params: Optional[OperationParameters] = None): persistence_diagram_features=PERSISTENCE_DIAGRAM_FEATURES ) self.data_transformer = None + self.save_pcd = False + + def __repr__(self): + return 'Topological Class for TS representation' def __evaluate_persistence_params(self, ts_data: np.array): if self.feature_extractor is None: @@ -67,12 +77,43 @@ def __evaluate_persistence_params(self, ts_data: np.array): persistence_diagram_extractor=persistence_diagram_extractor, persistence_diagram_features=PERSISTENCE_DIAGRAM_FEATURES) + def _generate_vr_mesh(self, pcd): + # Corresponding matrix of Euclidean pairwise distances + pairwise_distances = squareform(pdist(pcd)) + # Default parameter for ``metric`` is "euclidean" + vr_graph = VietorisRipsPersistence(metric="precomputed").fit_transform([pairwise_distances]) + return vr_graph + + def _generate_pcd(self, ts_data, persistence_params): + window_size_range = list(range(1, 35, 5)) + stride_range = list(range(1, 15, 3)) + pcd_params = list(product(window_size_range, stride_range)) + for params in pcd_params: + data_transformer = TopologicalTransformation(stride=params[1], persistence_params=persistence_params, + window_length=round(ts_data.shape[0] * 0.01 * params[0])) + point_cloud = data_transformer.time_series_to_point_cloud(input_data=ts_data, use_gtda=True) + # VR_mesh = self._generate_vr_mesh(point_cloud) + for scale in range(1, 15, 3): + numpy2stl(point_cloud, + f"./stl_scale_{scale}_ws_{params[0]}_stride_{params[1]}.stl", + max_width=300., + max_depth=200., + max_height=300., + scale=scale, + min_thickness_percent=0.5, + solid=False) + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(point_cloud) + o3d.io.write_point_cloud(f"./pcd_ws_{params[0]}_stride_{params[1]}.ply", pcd) + def _generate_features_from_ts(self, ts_data: np.array, persistence_params: dict) -> InputData: + if self.save_pcd: + self._generate_pcd(ts_data, persistence_params) if self.data_transformer is None: self.data_transformer = TopologicalTransformation( persistence_params=persistence_params, window_length=round(ts_data.shape[0] * 0.01 * self.window_size)) - point_cloud = self.data_transformer.time_series_to_point_cloud(input_data=ts_data) + point_cloud = self.data_transformer.time_series_to_point_cloud(input_data=ts_data, use_gtda=True) topological_features = self.feature_extractor.transform(point_cloud) topological_features = InputData(idx=np.arange(len(topological_features.values)), features=topological_features.values, diff --git a/tests/unit/core/models/test_quantile_extractor.py b/tests/unit/core/models/test_quantile_extractor.py index 7a04ec58f..15b837527 100644 --- a/tests/unit/core/models/test_quantile_extractor.py +++ b/tests/unit/core/models/test_quantile_extractor.py @@ -4,7 +4,7 @@ from fedot_ind.api.utils.data import init_input_data from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.models.quantile.quantile_extractor import QuantileExtractor +from fedot_ind.core.operation.transformation.representation.statistical.quantile_extractor import QuantileExtractor from fedot_ind.core.repository.constanst_repository import STAT_METHODS, STAT_METHODS_GLOBAL from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator diff --git a/tests/unit/core/models/test_recurrence_extractor.py b/tests/unit/core/models/test_recurrence_extractor.py index 6fdfc9c6e..bdb91981b 100644 --- a/tests/unit/core/models/test_recurrence_extractor.py +++ b/tests/unit/core/models/test_recurrence_extractor.py @@ -4,7 +4,7 @@ from fedot_ind.api.utils.data import init_input_data from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.models.recurrence.reccurence_extractor import RecurrenceExtractor +from fedot_ind.core.operation.transformation.representation.recurrence import RecurrenceExtractor from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator diff --git a/tests/unit/core/models/test_riemann_embeding.py b/tests/unit/core/models/test_riemann_embeding.py index e96aac7b5..98019197c 100644 --- a/tests/unit/core/models/test_riemann_embeding.py +++ b/tests/unit/core/models/test_riemann_embeding.py @@ -6,7 +6,7 @@ from fedot_ind.api.utils.data import init_input_data from fedot_ind.api.utils.path_lib import PATH_TO_DEFAULT_PARAMS -from fedot_ind.core.models.manifold.riemann_embeding import RiemannExtractor +from fedot_ind.core.operation.transformation.representation.manifold.riemann_embeding import RiemannExtractor from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator diff --git a/tests/unit/core/models/test_topological_extractor.py b/tests/unit/core/models/test_topological_extractor.py index 14893e35c..68da5d64f 100644 --- a/tests/unit/core/models/test_topological_extractor.py +++ b/tests/unit/core/models/test_topological_extractor.py @@ -1,9 +1,9 @@ import pytest from fedot.core.data.data import InputData, OutputData -from fedot_ind.api.utils.data import init_input_data +from fedot_ind.api.utils.data import init_input_data from fedot_ind.core.architecture.settings.computational import backend_methods as np -from fedot_ind.core.models.topological.topological_extractor import TopologicalExtractor +from fedot_ind.core.operation.transformation.representation.topological import TopologicalExtractor from fedot_ind.tools.synthetic.ts_datasets_generator import TimeSeriesDatasetsGenerator