diff --git a/examples/pipeline_example/time_series/ts_forecasting/transformer_forecasting.py b/examples/pipeline_example/time_series/ts_forecasting/transformer_forecasting.py index 5b9291d8c..d90f20a05 100644 --- a/examples/pipeline_example/time_series/ts_forecasting/transformer_forecasting.py +++ b/examples/pipeline_example/time_series/ts_forecasting/transformer_forecasting.py @@ -1,5 +1,8 @@ +import matplotlib.pyplot as plt import numpy as np +from fedot import Fedot from fedot.core.pipelines.pipeline_builder import PipelineBuilder +from fedot.core.repository.tasks import TsForecastingParams, Task, TaskTypesEnum from examples.example_utils import get_ts_data from examples.pipeline_example.time_series.ts_forecasting.ssa_forecasting import plot_metrics_and_prediction @@ -21,19 +24,17 @@ window_length = max(window_length_hac, window_length_heuristic) model_dict = { - # 'tst_model': - # PipelineBuilder().add_node('patch_tst_model', params={'patch_len': None, - # 'forecast_length': forecast_length, - # 'epochs': 200}), + 'tst_model': + PipelineBuilder().add_node('patch_tst_model', params={'patch_len': None, + 'forecast_length': forecast_length, + 'epochs': 10}), 'spectral_tst_model': PipelineBuilder().add_node('ar').add_node('eigen_basis', params={'window_size': window_length}, branch_idx=1). add_node('feature_filter_model', params={'grouping_level': 0.5}, branch_idx=1). add_node('patch_tst_model', params={'patch_len': None, 'forecast_length': forecast_length, - 'epochs': 100}, branch_idx=1).join_branches('cat_features', params={ - 'average_type': 'average', - 'prediction_length': forecast_length}), + 'epochs': 1}, branch_idx=1), 'baseline': PipelineBuilder().add_node('ar') } @@ -41,16 +42,32 @@ del model_dict['baseline'] baseline.fit(train_data) baseline_prediction = np.ravel(baseline.predict(test_data).predict) - - with IndustrialModels(): - for model in model_dict.keys(): - pipeline = model_dict[model].build() - pipeline.fit(train_data) - model_prediction = pipeline.predict(test_data).predict - plot_metrics_and_prediction(test_data, - train_data, - model_prediction, - baseline_prediction, - model, - dataset_name) - _ = 1 + IndustrialModels().setup_repository() + for model in model_dict.keys(): + # pipeline.fit(train_data) + model = Fedot(problem='ts_forecasting', + logging_level=10, + preset='ts', + available_operations=[ + 'ssa_forecaster', + 'patch_tst_model', + 'ar', + 'ridge', + 'ts_naive_average', + 'stl_arima', + 'lagged', 'arima', 'lasso' + ], + task_params=TsForecastingParams(forecast_length=forecast_length), + timeout=30 + ) + model.fit(train_data) + model.current_pipeline.show() + model_prediction = model.predict(test_data) + # + # model_prediction = pipeline.predict(test_data).predict + plot_metrics_and_prediction(test_data, + train_data, + model_prediction, + baseline_prediction, + model, + dataset_name) \ No newline at end of file diff --git a/fedot_ind/core/architecture/settings/constanst_repository.py b/fedot_ind/core/architecture/settings/constanst_repository.py index 10885e6cd..8591a820a 100644 --- a/fedot_ind/core/architecture/settings/constanst_repository.py +++ b/fedot_ind/core/architecture/settings/constanst_repository.py @@ -137,6 +137,7 @@ class TorchLossesConstant(Enum): CROSS_ENTROPY = nn.CrossEntropyLoss MULTI_CLASS_CROSS_ENTROPY = nn.BCEWithLogitsLoss MSE = nn.MSELoss + RMSE = RMSE SMAPE = SMAPELoss TWEEDIE_LOSS = TweedieLoss FOCAL_LOSS = FocalLoss @@ -177,6 +178,7 @@ class TorchLossesConstant(Enum): CROSS_ENTROPY = TorchLossesConstant.CROSS_ENTROPY.value MULTI_CLASS_CROSS_ENTROPY = TorchLossesConstant.MULTI_CLASS_CROSS_ENTROPY.value MSE = TorchLossesConstant.MSE.value +RMSE = TorchLossesConstant.RMSE.value SMAPE = TorchLossesConstant.SMAPE.value TWEEDIE_LOSS = TorchLossesConstant.TWEEDIE_LOSS.value FOCAL_LOSS = TorchLossesConstant.FOCAL_LOSS.value diff --git a/fedot_ind/core/models/nn/fedot_evaluation_strategy.py b/fedot_ind/core/models/nn/fedot_evaluation_strategy.py index 87afca1ef..b85c61b5d 100644 --- a/fedot_ind/core/models/nn/fedot_evaluation_strategy.py +++ b/fedot_ind/core/models/nn/fedot_evaluation_strategy.py @@ -2,6 +2,7 @@ from fedot.core.data.data import InputData, OutputData from fedot.core.operations.evaluation.evaluation_interfaces import EvaluationStrategy +from fedot.core.operations.evaluation.time_series import FedotTsForecastingStrategy from fedot.core.operations.operation_parameters import OperationParameters from fedot_ind.core.models.nn.network_impl.explainable_convolution_model import XCModel @@ -18,8 +19,7 @@ class FedotNNClassificationStrategy(EvaluationStrategy): 'omniscale_model': OmniScaleModel, 'tst_model': TSTModel, 'resnet_model': ResNetModel, - 'xcm_model': XCModel, - 'patch_tst_model': PatchTSTModel + 'xcm_model': XCModel } def _convert_to_operation(self, operation_type: str): @@ -45,3 +45,19 @@ def predict_for_fit(self, trained_operation, predict_data: InputData) -> OutputD return trained_operation.predict_for_fit(predict_data, self.output_mode) else: return trained_operation.predict(predict_data, self.output_mode) + + +class FedotNNTimeSeriesStrategy(FedotTsForecastingStrategy): + __operations_by_types = { + 'patch_tst_model': PatchTSTModel + } + + def _convert_to_operation(self, operation_type: str): + if operation_type in self.__operations_by_types.keys(): + return self.__operations_by_types[operation_type] + else: + raise ValueError(f'Impossible to obtain custom preprocessing strategy for {operation_type}') + + def __init__(self, operation_type: str, params: Optional[OperationParameters] = None): + self.operation_impl = self._convert_to_operation(operation_type) + super().__init__(operation_type, params) \ No newline at end of file diff --git a/fedot_ind/core/models/nn/network_impl/patch_tst.py b/fedot_ind/core/models/nn/network_impl/patch_tst.py index 02bbcb242..6ad3c7bb3 100644 --- a/fedot_ind/core/models/nn/network_impl/patch_tst.py +++ b/fedot_ind/core/models/nn/network_impl/patch_tst.py @@ -16,12 +16,11 @@ from fedot_ind.core.architecture.abstraction.decorators import convert_inputdata_to_torch_time_series_dataset from fedot_ind.core.architecture.preprocessing.data_convertor import DataConverter from fedot_ind.core.architecture.settings.computational import default_device -from fedot_ind.core.architecture.settings.constanst_repository import MSE, SMAPE +from fedot_ind.core.architecture.settings.constanst_repository import RMSE, MSE, SMAPE from fedot_ind.core.models.nn.network_impl.base_nn_model import BaseNeuralModel from fedot_ind.core.models.nn.network_modules.layers.backbone import _PatchTST_backbone from fedot_ind.core.models.nn.network_modules.layers.special import SeriesDecomposition, \ EarlyStopping, adjust_learning_rate -from fedot_ind.core.operation.transformation.basis.fourier import FourierBasisImplementation from fedot_ind.core.operation.transformation.data.hankel import HankelMatrix from torch.optim import lr_scheduler @@ -55,58 +54,30 @@ def __init__(self, norm='BatchNorm', # type of normalization layer used in the encoder pre_norm=False, # flag to indicate if normalization is applied as the first step in the sublayers res_attention=True, # flag to indicate if Residual MultiheadAttention should be used - store_attn=False, # can be used to visualize attention weights + store_attn=True, + preprocess_to_lagged=False # can be used to visualize attention weights ): - super().__init__() # model if pred_dim is None: pred_dim = seq_len - self.decomposition = decomposition - if self.decomposition: - self.decomp_module = SeriesDecomposition(kernel_size) - self.model_trend = _PatchTST_backbone(input_dim=input_dim, seq_len=seq_len, pred_dim=pred_dim, - patch_len=patch_len, stride=stride, n_layers=n_layers, - d_model=d_model, - n_heads=n_heads, d_ff=d_ff, norm=norm, attn_dropout=attn_dropout, - dropout=dropout, act=activation, res_attention=res_attention, - pre_norm=pre_norm, - store_attn=store_attn, padding_patch=padding_patch, - individual=individual, revin=revin, affine=affine, - subtract_last=subtract_last) - self.model_res = _PatchTST_backbone(input_dim=input_dim, seq_len=seq_len, pred_dim=pred_dim, - patch_len=patch_len, stride=stride, n_layers=n_layers, d_model=d_model, - n_heads=n_heads, d_ff=d_ff, norm=norm, attn_dropout=attn_dropout, - dropout=dropout, act=activation, res_attention=res_attention, - pre_norm=pre_norm, - store_attn=store_attn, padding_patch=padding_patch, - individual=individual, revin=revin, affine=affine, - subtract_last=subtract_last) - self.patch_num = self.model_trend.patch_num - else: - self.model = _PatchTST_backbone(input_dim=input_dim, seq_len=seq_len, pred_dim=pred_dim, - patch_len=patch_len, stride=stride, n_layers=n_layers, d_model=d_model, - n_heads=n_heads, d_ff=d_ff, norm=norm, attn_dropout=attn_dropout, - dropout=dropout, act=activation, res_attention=res_attention, - pre_norm=pre_norm, - store_attn=store_attn, padding_patch=padding_patch, - individual=individual, revin=revin, affine=affine, - subtract_last=subtract_last) - self.patch_num = self.model.patch_num + self.model = _PatchTST_backbone(input_dim=input_dim, seq_len=seq_len, pred_dim=pred_dim, + patch_len=patch_len, stride=stride, n_layers=n_layers, d_model=d_model, + n_heads=n_heads, d_ff=d_ff, norm=norm, attn_dropout=attn_dropout, + dropout=dropout, act=activation, res_attention=res_attention, + pre_norm=pre_norm, + store_attn=store_attn, padding_patch=padding_patch, + individual=individual, revin=revin, affine=affine, + subtract_last=subtract_last, preprocess_to_lagged=preprocess_to_lagged) + self.patch_num = self.model.patch_num def forward(self, x): """Args: x: rank 3 tensor with shape [batch size x features x sequence length] """ - if self.decomposition: - res_init, trend_init = self.decomp_module(x) - res = self.model_res(res_init) - trend = self.model_trend(trend_init) - x = res + trend - else: - x = self.model(x) + x = self.model(x) return x @@ -135,47 +106,76 @@ class PatchTSTModel(BaseNeuralModel): """ def __init__(self, params: Optional[OperationParameters] = {}): - self.epochs = params.get('epochs', 100) + self.epochs = params.get('epochs', 1) self.batch_size = params.get('batch_size', 16) self.learning_rate = params.get('learning_rate', 0.001) self.use_amp = params.get('use_amp', False) - self.horizon = params.get('forecast_length', 30) + self.horizon = params.get('forecast_length', None) self.patch_len = params.get('patch_len', None) self.output_attention = params.get('output_attention', False) self.test_patch_len = self.patch_len + self.preprocess_to_lagged = False + self.forecast_mode = params.get('forecast_mode', 'out_of_sample') self.model_list = [] def _init_model(self, ts): model = PatchTST(input_dim=1, output_dim=None, - seq_len=ts.features.shape[0], - pred_dim=self.horizon).to(default_device()) + seq_len=self.seq_len, + pred_dim=self.horizon, + patch_len=self.patch_len, + preprocess_to_lagged=self.preprocess_to_lagged).to(default_device()) optimizer = optim.Adam(model.parameters(), lr=self.learning_rate) - loss_fn = SMAPE() + loss_fn = RMSE() return model, loss_fn, optimizer - def _fit_model(self, ts: InputData, split_data: bool = True): - if len(ts.features.shape) == 1: - ts.features = ts.features.reshape(1, -1) - - for index, sequences in enumerate(ts.features): + def _fit_model(self, input_data: InputData, split_data: bool = True): + if self.preprocess_to_lagged: + self.patch_len = input_data.features.shape[1] + train_loader = self.__create_torch_loader(input_data) + else: if self.patch_len is None: - dominant_window_size = WindowSizeSelector(method='dff').get_window_size(sequences) - patch_len = 2 * dominant_window_size - self.test_patch_len = patch_len - train_loader = self._prepare_data(sequences, patch_len, False) - model, loss_fn, optimizer = self._init_model(ts) - model = self._train_loop(model, train_loader, loss_fn, optimizer) - self.model_list.append(model) + dominant_window_size = WindowSizeSelector(method='dff').get_window_size(input_data.features) + self.patch_len = 2 * dominant_window_size + train_loader = self._prepare_data(input_data.features, self.patch_len, False) + self.test_patch_len = self.patch_len + model, loss_fn, optimizer = self._init_model(input_data) + model = self._train_loop(model, train_loader, loss_fn, optimizer) + self.model_list.append(model) + + def __preprocess_for_fedot(self, input_data): + input_data.features = np.squeeze(input_data.features) + if self.horizon is None: + self.horizon = input_data.task.task_params.forecast_length + if input_data.features.shape[1] != 1: + self.preprocess_to_lagged = True + if len(input_data.features.shape) == 1: + input_data.features = input_data.features.reshape(1, -1) + if self.preprocess_to_lagged: + self.seq_len = input_data.features.shape[0] + input_data.features.shape[1] + else: + self.seq_len = input_data.features.shape[0] + self.target = input_data.target + self.task_type = input_data.task + return input_data + + def __create_torch_loader(self, train_data): + + train_dataset = self._create_dataset(train_data) + if not self.preprocess_to_lagged: + train_dataset.x = train_dataset.x.permute(0, 2, 1) + train_dataset.y = train_dataset.y.permute(0, 2, 1) + train_loader = torch.utils.data.DataLoader( + data.TensorDataset(train_dataset.x, train_dataset.y), + batch_size=self.batch_size, shuffle=False) + return train_loader def fit(self, input_data: InputData): """ Method for feature generation for all series """ - input_data.features = np.squeeze(input_data.features) - self.target = input_data.target - self.task_type = input_data.task + input_data = self.__preprocess_for_fedot(input_data) self._fit_model(input_data) def split_data(self, input_data): @@ -205,7 +205,8 @@ def _create_dataset(self, freq: int = 1): return ts - def _prepare_data(self, ts, + def _prepare_data(self, + ts, patch_len, split_data: bool = True, validation_blocks: int = None): @@ -222,14 +223,7 @@ def _prepare_data(self, ts, _, train_data.features, train_data.target = transform_features_and_target_into_lagged(train_data, self.horizon, patch_len) - - train_dataset = self._create_dataset(train_data) - train_dataset.x = train_dataset.x.permute(0, 2, 1) - train_dataset.y = train_dataset.y.permute(0, 2, 1) - train_loader = torch.utils.data.DataLoader( - data.TensorDataset(train_dataset.x, train_dataset.y), - batch_size=self.batch_size, shuffle=False) - + train_loader = self.__create_torch_loader(train_data) return train_loader def _train_loop(self, model, @@ -278,24 +272,24 @@ def _train_loop(self, model, return model - def _predict(self, model, test_data): - features = HankelMatrix(time_series=test_data.T, - window_size=self.test_patch_len).trajectory_matrix[:, -1:] - features = torch.from_numpy(DataConverter(data=features).convert_to_torch_format()).float().permute(2, 1, 0) - target = torch.from_numpy(DataConverter(data=features).convert_to_torch_format()).float() - test_loader = torch.utils.data.DataLoader(data.TensorDataset(features, target), - batch_size=1, shuffle=False) + def _predict(self, model, test_loader): model.eval() - with torch.no_grad(): - for i, (batch_x, batch_y) in enumerate(test_loader): - batch_x = batch_x.float().to(default_device()) - batch_y = batch_y.float().to(default_device()) - # decoder input - dec_inp = torch.zeros_like(batch_y[:, :, :]).float() - dec_inp = torch.cat([batch_y[:, :, :], dec_inp], dim=1).float().to(default_device()) - # encoder - decoder - outputs = model(batch_x) + if self.forecast_mode == 'in_sample': + for i, (batch_x, batch_y) in enumerate(test_loader): + batch_x = batch_x.float().to(default_device()) + batch_y = batch_y.float().to(default_device()) + # decoder input + dec_inp = torch.zeros_like(batch_y[:, :, :]).float() + dec_inp = torch.cat([batch_y[:, :, :], dec_inp], dim=1).float().to(default_device()) + # encoder - decoder + outputs = model(batch_x) + else: + last_patch = test_loader.dataset[0][-1] + c, s = last_patch.size() + last_patch = last_patch.reshape(1, c, s).to(default_device()) + outputs = model(last_patch) + return outputs.flatten().cpu().numpy() def _encoder_decoder_transition(self, batch_x, batch_x_mark, dec_inp, batch_y_mark): @@ -312,7 +306,10 @@ def _encoder_decoder_transition(self, batch_x, batch_x_mark, dec_inp, batch_y_ma def predict(self, test_data, output_mode: str = 'labels'): - y_pred = self._predict_loop(test_data) + y_pred = [] + for model in self.model_list: + y_pred.append(self._predict_loop(model, test_data)) + y_pred = np.array(y_pred) forecast_idx_predict = np.arange(start=test_data.idx[-self.horizon], stop=test_data.idx[-self.horizon] + len(y_pred), step=1) @@ -327,7 +324,10 @@ def predict(self, def predict_for_fit(self, test_data, output_mode: str = 'labels'): - y_pred = self._predict_loop(test_data) + y_pred = [] + for model in self.model_list: + y_pred.append(self._predict_loop(model, test_data)) + y_pred = np.array(y_pred) forecast_idx_predict = np.arange(len(y_pred)) predict = OutputData( idx=forecast_idx_predict, @@ -337,18 +337,25 @@ def predict_for_fit(self, data_type=DataTypesEnum.table) return predict - def _predict_loop(self, + def _predict_loop(self, model, test_data): - y_pred = [] test_data.features = test_data.features.squeeze() if len(test_data.features.shape) == 1: test_data.features = test_data.features.reshape(1, -1) - for sequences, model in zip(test_data.features, self.model_list): - if type(model) is np.ndarray: - y_pred.append(model) - else: - y_pred.append(self._predict(model, sequences)) - y_pred = np.sum(np.array(y_pred), axis=0) - return y_pred + if not self.preprocess_to_lagged: + features = HankelMatrix(time_series=test_data.T, + window_size=self.test_patch_len).trajectory_matrix[:, -1:] + features = torch.from_numpy(DataConverter(data=features). + convert_to_torch_format()).float().permute(2, 1, 0) + target = torch.from_numpy(DataConverter(data=features).convert_to_torch_format()).float() + + else: + features = test_data.features + features = torch.from_numpy(DataConverter(data=features). + convert_to_torch_format()).float() + target = torch.from_numpy(DataConverter(data=features).convert_to_torch_format()).float() + test_loader = torch.utils.data.DataLoader(data.TensorDataset(features, target), + batch_size=self.batch_size, shuffle=False) + return self._predict(model, test_loader) diff --git a/fedot_ind/core/models/nn/network_modules/layers/attention_layers.py b/fedot_ind/core/models/nn/network_modules/layers/attention_layers.py index fc0e07b6e..0c179efab 100644 --- a/fedot_ind/core/models/nn/network_modules/layers/attention_layers.py +++ b/fedot_ind/core/models/nn/network_modules/layers/attention_layers.py @@ -68,8 +68,15 @@ def forward(self, q: Tensor, k: Tensor, v: Tensor, prev: Optional[Tensor] = None class MultiHeadAttention(Module): - def __init__(self, d_model, n_heads, d_k=None, d_v=None, res_attention=False, attn_dropout=0., proj_dropout=0., - qkv_bias=True, lsa=False): + def __init__(self, d_model, + n_heads, + d_k=None, + d_v=None, + res_attention=False, + attn_dropout=0., + proj_dropout=0., + qkv_bias=True, + lsa=False): """Multi Head Attention Layer Input shape: diff --git a/fedot_ind/core/models/nn/network_modules/layers/backbone.py b/fedot_ind/core/models/nn/network_modules/layers/backbone.py index b5e54f3a8..e4e3eb92a 100644 --- a/fedot_ind/core/models/nn/network_modules/layers/backbone.py +++ b/fedot_ind/core/models/nn/network_modules/layers/backbone.py @@ -12,7 +12,7 @@ def __init__(self, input_dim, seq_len, pred_dim, patch_len, stride, d_ff=256, norm='BatchNorm', attn_dropout=0., dropout=0., act="gelu", res_attention=True, pre_norm=False, store_attn=False, padding_patch=True, individual=False, - revin=True, affine=True, subtract_last=False): + revin=True, affine=True, subtract_last=False, preprocess_to_lagged=False): super().__init__() @@ -27,6 +27,7 @@ def __init__(self, input_dim, seq_len, pred_dim, patch_len, stride, patch_num = int((seq_len - patch_len) / stride + 1) + 1 self.patch_num = patch_num self.padding_patch_layer = nn.ReplicationPad1d((stride, 0)) # original padding at the end + self.preprocess_to_lagged = preprocess_to_lagged # Unfold self.unfold = nn.Unfold(kernel_size=(1, patch_len), stride=stride) @@ -55,14 +56,18 @@ def forward(self, z: Tensor): z = self.revin_layer(z, torch.tensor(True, dtype=torch.bool)) # do patching - z = self.padding_patch_layer(z) - b, c, s = z.size() - z = z.reshape(-1, 1, 1, s) - z = self.unfold(z) - z = z.permute(0, 2, 1).reshape(b, c, -1, self.patch_len).permute(0, 1, 3, 2) + if not self.preprocess_to_lagged: + z = self.padding_patch_layer(z) + b, c, s = z.size() + z = z.reshape(-1, 1, 1, s) + z = self.unfold(z) + z = z.permute(0, 2, 1).reshape(b, c, -1, self.patch_len) + else: + b, c, s = z.size() + z = z.reshape(-1, 1, 1, s).permute(0, 1, 3, 2) # model - z = self.backbone(z) # z: [bs x nvars x d_model x patch_num] + z, scores = self.backbone(z) # z: [bs x nvars x d_model x patch_num] z = self.head(z) # z: [bs x nvars x pred_dim] # denorm diff --git a/fedot_ind/core/models/nn/network_modules/layers/linear_layers.py b/fedot_ind/core/models/nn/network_modules/layers/linear_layers.py index 8cf2f426f..a00fe7921 100644 --- a/fedot_ind/core/models/nn/network_modules/layers/linear_layers.py +++ b/fedot_ind/core/models/nn/network_modules/layers/linear_layers.py @@ -267,7 +267,8 @@ def forward(self, x: Tensor): return self.layer(x) except Exception: self.layer = nn.Sequential(nn.Flatten(start_dim=-2), - nn.Linear(x.shape[3]*self.nf, self.pred_dim)).to(default_device()) + nn.Linear(x.shape[3] * self.nf, self.pred_dim, device=default_device())) return self.layer(x) + Noop = nn.Sequential() diff --git a/fedot_ind/core/models/nn/network_modules/layers/special.py b/fedot_ind/core/models/nn/network_modules/layers/special.py index bc67bb7c3..c907f0d7e 100644 --- a/fedot_ind/core/models/nn/network_modules/layers/special.py +++ b/fedot_ind/core/models/nn/network_modules/layers/special.py @@ -6,13 +6,14 @@ from torch import nn, optim, Tensor from fedot_ind.core.architecture.abstraction.decorators import convert_to_torch_tensor +from fedot_ind.core.architecture.settings.computational import default_device from fedot_ind.core.models.nn.network_modules.activation import get_activation_fn from fedot_ind.core.models.nn.network_modules.layers.attention_layers import MultiHeadAttention from fedot_ind.core.models.nn.network_modules.layers.conv_layers import Conv1d, ConvBlock from fedot_ind.core.models.nn.network_modules.layers.linear_layers import BN1d, Concat, Add, Noop, Transpose from fastcore.meta import delegates -from fastai.torch_core import Module, default_device +from fastai.torch_core import Module class EarlyStopping: @@ -283,10 +284,12 @@ def forward(self, src: Tensor, prev: Optional[Tensor] = None): # Feed-forward sublayer if self.pre_norm: src = self.norm_ffn(src) + ## Position-wise Feed-Forward src2 = self.ff(src) ## Add & Norm src = src + self.dropout_ffn(src2) # Add: residual connection with residual dropout + if not self.pre_norm: src = self.norm_ffn(src) @@ -295,6 +298,7 @@ def forward(self, src: Tensor, prev: Optional[Tensor] = None): else: return src + class _TSTiEncoder(nn.Module): # i means channel-independent def __init__(self, input_dim, patch_num, patch_len, n_layers=3, d_model=128, n_heads=16, d_k=None, d_v=None, d_ff=256, norm='BatchNorm', attn_dropout=0., dropout=0., act="gelu", store_attn=False, @@ -341,15 +345,15 @@ def forward(self, x: Tensor): try: pos_encooding = x + self.W_pos except Exception: - W_pos = torch.empty((x.shape[1], x.shape[2])) + W_pos = torch.empty((x.shape[1], x.shape[2]), device=default_device()) nn.init.uniform_(W_pos, -0.02, 0.02) - self.W_pos = nn.Parameter(W_pos).to(default_device()) - pos_encooding = x + self.W_pos # x: [bs * nvars x patch_num x d_model] + self.W_pos = nn.Parameter(W_pos) + pos_encooding = x + self.W_pos # x: [bs * nvars x patch_num x d_model] x = self.dropout(pos_encooding) # x: [bs * nvars x patch_num x d_model] + scores = None # Encoder if self.res_attention: - scores = None for mod in self.layers: x, scores = mod(x, prev=scores) else: @@ -357,7 +361,7 @@ def forward(self, x: Tensor): x = torch.reshape(x, (-1, n_vars, x.shape[-2], x.shape[-1])) # x: [bs x nvars x patch_num x d_model] x = x.permute(0, 1, 3, 2) # x: [bs x nvars x d_model x patch_num] - return x + return x, scores class RevIN(nn.Module): diff --git a/fedot_ind/core/models/nn/network_modules/losses.py b/fedot_ind/core/models/nn/network_modules/losses.py index 81f552d98..7f011aba4 100644 --- a/fedot_ind/core/models/nn/network_modules/losses.py +++ b/fedot_ind/core/models/nn/network_modules/losses.py @@ -169,3 +169,13 @@ def __init__(self): def forward(self, input: Tensor, target: Tensor) -> Tensor: return 100 * torch.mean(2 * torch.abs(input - target) / (torch.abs(target) + torch.abs(input)) + 1e-8) + + +class RMSE(Module): + def __init__(self): + super().__init__() + + def forward(self, input: Tensor, target: Tensor) -> Tensor: + criterion = nn.MSELoss() + loss = torch.sqrt(criterion(input, target)) + return loss \ No newline at end of file diff --git a/fedot_ind/core/operation/dummy/dummy_operation.py b/fedot_ind/core/operation/dummy/dummy_operation.py index 77766ae7d..b4331601c 100644 --- a/fedot_ind/core/operation/dummy/dummy_operation.py +++ b/fedot_ind/core/operation/dummy/dummy_operation.py @@ -1,14 +1,25 @@ +from typing import Optional + +import numpy as np from fedot.core.data.data import InputData, OutputData from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import \ DataOperationImplementation +from fedot.core.operations.operation_parameters import OperationParameters from fedot.core.repository.dataset_types import DataTypesEnum class DummyOperation(DataOperationImplementation): - + def __init__(self, params: Optional[OperationParameters] = None): + super().__init__(params) + self.average = params.get('average_type', None) + self.prediction_length = params.get('prediction_length', None) def fit(self, input_data: InputData): pass def transform(self, input_data: InputData) -> OutputData: - predict = self._convert_to_output(input_data, input_data.features, data_type=DataTypesEnum.table) + if self.average is not None: + transformed_features = np.average(input_data.features.reshape(-1, self.prediction_length), axis=0) + else: + transformed_features = input_data.features + predict = self._convert_to_output(input_data, transformed_features, data_type=DataTypesEnum.table) return predict diff --git a/fedot_ind/core/repository/data/industrial_data_operation_repository.json b/fedot_ind/core/repository/data/industrial_data_operation_repository.json index be37f9534..e6595a547 100644 --- a/fedot_ind/core/repository/data/industrial_data_operation_repository.json +++ b/fedot_ind/core/repository/data/industrial_data_operation_repository.json @@ -129,8 +129,24 @@ ], "tasks": "[TaskTypesEnum.classification]" }, - "industrial_preprocessing": { - "tasks": "[TaskTypesEnum.classification, TaskTypesEnum.ts_forecasting]", + "industrial_preprocessing_classification": { + "tasks": "[TaskTypesEnum.classification,]", + "input_type": "[DataTypesEnum.image, DataTypesEnum.table]", + "output_type": "[DataTypesEnum.image, DataTypesEnum.table]", + "accepted_node_types": [ + "any" + ], + "forbidden_node_types": "[]", + "strategies": [ + "fedot_ind.core.operation.interfaces.industrial_preprocessing_strategy", + "IndustrialPreprocessingStrategy" + ], + "tags": [ + ], + "description": "Implementations of industrial preprocessing operations" + }, + "industrial_preprocessing_forecasting": { + "tasks": "[TaskTypesEnum.ts_forecasting]", "input_type": "[DataTypesEnum.image, DataTypesEnum.table]", "output_type": "[DataTypesEnum.image, DataTypesEnum.table]", "accepted_node_types": [ @@ -148,7 +164,7 @@ }, "operations": { "eigen_basis": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -157,7 +173,7 @@ ] }, "wavelet_basis": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -166,7 +182,7 @@ ] }, "fourier_basis": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -175,16 +191,16 @@ ] }, "topological_extractor": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_forecasting", "presets": [ - "fast_train" + "fast_train","best_quality","ts" ], "tags": [ "extractor" ] }, "quantile_extractor": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -193,7 +209,7 @@ ] }, "signal_extractor": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -202,7 +218,7 @@ ] }, "recurrence_extractor": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -210,8 +226,16 @@ "extractor" ] }, + "topological_features": { + "meta": "custom_preprocessing", + "presets": [ + "ts" + ], + "tags": [ + "non_applicable_for_ts" + ]}, "minirocket_extractor": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -220,7 +244,7 @@ ] }, "feature_filter_model": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], @@ -229,7 +253,7 @@ ] }, "cat_features": { - "meta": "industrial_preprocessing", + "meta": "industrial_preprocessing_classification", "presets": [ "fast_train" ], diff --git a/fedot_ind/core/repository/data/industrial_model_repository.json b/fedot_ind/core/repository/data/industrial_model_repository.json index f09b5dec3..376af3f64 100644 --- a/fedot_ind/core/repository/data/industrial_model_repository.json +++ b/fedot_ind/core/repository/data/industrial_model_repository.json @@ -150,7 +150,7 @@ "description": "Implementations of fedot automl classification strategy" }, "fedot_NN_classification": { - "tasks": "[TaskTypesEnum.classification, TaskTypesEnum.ts_forecasting]", + "tasks": "[TaskTypesEnum.classification]", "input_type": "[DataTypesEnum.image, DataTypesEnum.table]", "output_type": "[DataTypesEnum.table, DataTypesEnum.image]", "accepted_node_types": [ @@ -165,6 +165,22 @@ ], "description": "Implementations of fedot NN classification strategy" }, + "fedot_NN_forecasting": { + "tasks": "[TaskTypesEnum.ts_forecasting]", + "input_type": "[DataTypesEnum.image, DataTypesEnum.table]", + "output_type": "[DataTypesEnum.table, DataTypesEnum.image]", + "accepted_node_types": [ + "any" + ], + "forbidden_node_types": "[]", + "strategies": [ + "fedot_ind.core.models.nn.fedot_evaluation_strategy", + "FedotNNTimeSeriesStrategy" + ], + "tags": ["time_series" + ], + "description": "Implementations of fedot NN classification strategy" + }, "fedot_automl_regression": { "tasks": "[TaskTypesEnum.regression]", "input_type": "[DataTypesEnum.table]", @@ -186,9 +202,7 @@ "operations": { "ssa_forecaster": { "meta": "industrial_decomposition_for_forecasting", - "presets": [ - "fast_train" - ], + "presets": ["best_quality", "ts"], "tags": [ "decomposition" ] @@ -257,13 +271,12 @@ ] }, "patch_tst_model": { - "meta": "fedot_NN_classification", - "presets": [ - "best_quality" - ], + "meta": "fedot_NN_forecasting", + "presets": ["fast_train", "best_quality", "ts"], "tags": [ - "automl" - ] + "deep", "ts_model" + ], + "input_type": "[DataTypesEnum.ts]" }, "adareg": { "meta": "sklearn_regr", diff --git a/fedot_ind/core/repository/initializer_industrial_models.py b/fedot_ind/core/repository/initializer_industrial_models.py index 03db63322..99a2c1f5f 100644 --- a/fedot_ind/core/repository/initializer_industrial_models.py +++ b/fedot_ind/core/repository/initializer_industrial_models.py @@ -107,6 +107,22 @@ def __init__(self): 'industrial_model_repository.json') self.base_model_path = pathlib.Path('model_repository.json') + def setup_repository(self): + OperationTypesRepository.__repository_dict__.update( + {'data_operation': {'file': self.industrial_data_operation_path, + 'initialized_repo': None, + 'default_tags': []}}) + + OperationTypesRepository.assign_repo('data_operation', self.industrial_data_operation_path) + + OperationTypesRepository.__repository_dict__.update( + {'model': {'file': self.industrial_model_path, + 'initialized_repo': None, + 'default_tags': []}}) + OperationTypesRepository.assign_repo('model', self.industrial_model_path) + + setattr(PipelineSearchSpace, "get_parameters_dict", get_industrial_search_space) + def __enter__(self): """ Switching to industrial models diff --git a/fedot_ind/core/tuning/search_space.py b/fedot_ind/core/tuning/search_space.py index 62084277b..d7cc5d030 100644 --- a/fedot_ind/core/tuning/search_space.py +++ b/fedot_ind/core/tuning/search_space.py @@ -30,6 +30,9 @@ 'sampling-scope': [['mexh', 'shan', 'morl', 'cmor', 'fbsp', 'db5', 'sym5']]}}, 'minirocket_extractor': {'num_features': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(500, 10000, 500)]]}}, + 'patch_tst_model': + {'epochs': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(10, 100, 10)]]}, + 'batch_size': {'hyperopt-dist': hp.choice, 'sampling-scope': [[x for x in range(8, 64, 6)]]}}, 'ssa_forecaster': {'window_size_method': {'hyperopt-dist': hp.choice, 'window_size_method': [['hac', 'dff', 'mwf', 'sss']]}} }