refactor minirocket for multichanel, add early stop

examples/pipeline_example/time_series/ts_classification/basic_example.py

@@ -116,9 +116,9 @@
         'SelfRegulationSCP2',
         'StandWalkJump',
     ]
-    error_model = PipelineBuilder().add_node('resample').add_node('resample', branch_idx=1) \
-        .add_node('minirocket_extractor', branch_idx=1).add_node('quantile_extractor', branch_idx=1).join_branches(
-        'logit').build()
+    # error_model = PipelineBuilder().add_node('resample').add_node('resample', branch_idx=1) \
+    #     .add_node('minirocket_extractor', branch_idx=1).add_node('quantile_extractor', branch_idx=1).join_branches(
+    #     'logit').build()
     #error_model = PipelineBuilder().add_node('logit').add_node('logit').build()
     # error_model = PipelineBuilder().add_node('pca').add_node('resample', branch_idx=1).add_node('quantile_extractor', branch_idx=1).join_branches(
     #     'logit').build()
@@ -135,15 +135,15 @@
                       num_of_generations=20,
                       optimizer=IndustrialEvoOptimizer,
                       available_operations=ts_clf_operations,
-                      timeout=60,
+                      timeout=30,
                       with_tuning=False
                       )
-        model = error_model
+        #model = error_model
         model.fit(input_data)
-        features = model.predict(val_data, 'labels')
+        features = model.predict(val_data)
         metric = evaluate_metric(target=val_data.target, prediction=features)
         try:
-            acc = accuracy_score(y_true=val_data.target, y_pred=features)
+            acc = accuracy_score(y_true=val_data.target, y_pred=features.predict)
         except Exception:
             acc = accuracy_score(y_true=val_data.target, y_pred=np.argmax(features, axis=1))
         metric_dict.update({model: metric})

examples/pipeline_example/time_series/ts_forecasting/transformer_forecasting.py

@@ -11,7 +11,7 @@
 
 if __name__ == '__main__':
 
-    forecast_length = 14
+    forecast_length = 16
     window_size_percentage = 10
 
     train_data, test_data, dataset_name = get_ts_data('m4_daily',
@@ -27,7 +27,7 @@
         'tst_model':
             PipelineBuilder().add_node('patch_tst_model', params={'patch_len': None,
                                                                   'forecast_length': forecast_length,
-                                                                  'epochs': 10}),
+                                                                  'epochs': 50}),
         'spectral_tst_model':
             PipelineBuilder().add_node('ar').add_node('eigen_basis',
                                                       params={'window_size': window_length}, branch_idx=1).
@@ -45,7 +45,8 @@
     error_pipeline = PipelineBuilder().add_node('lagged').add_node('ssa_forecaster').add_node('ts_naive_average',
                                                                                                branch_idx=1).join_branches('lasso').build()
     for model in model_dict.keys():
-        # pipeline.fit(train_data)
+        pipeline = model_dict[model].build()
+        pipeline.fit(train_data)
         model = Fedot(problem='ts_forecasting',
                       logging_level=20,
                       n_jobs=1,
@@ -64,6 +65,7 @@
                       task_params=TsForecastingParams(forecast_length=forecast_length),
                       timeout=180
                       )
+        preds = pipeline.predict(test_data)
         error_pipeline.fit(train_data)
         model.fit(train_data)
         model.current_pipeline.show()

fedot_ind/core/models/nn/network_impl/base_nn_model.py

@@ -140,8 +140,8 @@ def _train_loop(self, train_loader, val_loader, loss_fn, optimizer):
                 valid_loss /= len(val_loader.dataset)
                 print('Epoch: {},Validation Loss: {:.2f}'.format(epoch,
                                                                  valid_loss))
-
-            adjust_learning_rate(optimizer, scheduler, epoch + 1, args, printout=False)
+            early_stopping(training_loss, self.model, './')
+            adjust_learning_rate(optimizer, scheduler, epoch + 1, self.learning_rate, printout=False)
             scheduler.step()
 
             if early_stopping.early_stop:
@@ -150,7 +150,7 @@ def _train_loop(self, train_loader, val_loader, loss_fn, optimizer):
             print('Updating learning rate to {}'.format(scheduler.get_last_lr()[0]))
 
     @convert_to_3d_torch_array
-    def _fit_model(self, ts: InputData, split_data: bool = True):
+    def _fit_model(self, ts: InputData, split_data: bool = False):
         self._train_loop(*self._prepare_data(ts, split_data), *self._init_model(ts))
 
     @convert_to_3d_torch_array

fedot_ind/core/models/nn/network_impl/mini_rocket.py

@@ -71,6 +71,7 @@ def fit(self, X, chunksize=None):
         else:
             self(X[idxs].to(self.kernels.device))
         self.fitting = False
+        return self
 
     def forward(self, x):
         _features = []
@@ -274,22 +275,36 @@ def __init__(self, params: Optional[OperationParameters] = None):
     def __repr__(self):
         return 'LargeFeatureSpace'
 
-    def _generate_features_from_ts(self, ts: np.array):
-        mrf = MiniRocketFeatures(input_dim=ts.shape[1],
-                                 seq_len=ts.shape[2],
-                                 num_features=self.num_features).to(default_device())
+    def _generate_features_from_ts(self, ts: np.array, mode: str = 'multivariate'):
 
-        mrf.fit(ts)
-        features = get_minirocket_features(ts, mrf)
-        features = features.swapaxes(1,2)
-        minirocket_features = OutputData(idx=np.arange(len(features)),
+        if ts.shape[1] > 1 and mode == 'chanel_independent':
+            mrf = MiniRocketFeatures(input_dim=1,
+                                     seq_len=ts.shape[2],
+                                     num_features=self.num_features).to(default_device())
+
+            n_dim = range(ts.shape[1])
+            ts_converted = [ts[:, i, :] for i in n_dim]
+            ts_converted = [x.reshape(x.shape[0], 1, x.shape[1]) for x in ts_converted]
+            model_list = [mrf for i in n_dim]
+        else:
+            mrf = MiniRocketFeatures(input_dim=ts.shape[1],
+                                     seq_len=ts.shape[2],
+                                     num_features=self.num_features).to(default_device())
+
+            ts_converted = [ts]
+            model_list = [mrf]
+
+        fitted_model = [model.fit(data) for model, data in zip(model_list, ts_converted)]
+        features = [get_minirocket_features(data, model) for model, data in zip(fitted_model, ts_converted)]
+        minirocket_features = [feature_by_dim.swapaxes(1, 2) for feature_by_dim in features]
+        minirocket_features = np.concatenate(minirocket_features, axis=1)
+        minirocket_features = OutputData(idx=np.arange(minirocket_features.shape[2]),
                                          task=self.task,
-                                         predict=features,
-                                         data_type=DataTypesEnum.table)
+                                         predict=minirocket_features,
+                                         data_type=DataTypesEnum.image)
         return minirocket_features
 
     def generate_minirocket_features(self, ts: np.array) -> InputData:
-
         return self._generate_features_from_ts(ts)
 
     @convert_to_3d_torch_array

fedot_ind/core/models/nn/network_modules/layers/special.py

@@ -17,7 +17,7 @@
 
 
 class EarlyStopping:
-    def __init__(self, patience=7, verbose=False, delta=0):
+    def __init__(self, patience=5, verbose=False, delta=0):
         self.patience = patience
         self.verbose = verbose
         self.counter = 0
@@ -48,28 +48,28 @@ def save_checkpoint(self, val_loss, model, path):
         self.val_loss_min = val_loss
 
 
-def adjust_learning_rate(optimizer, scheduler, epoch, args, printout=True):
+def adjust_learning_rate(optimizer, scheduler, epoch, learning_rate, printout=True, lradj = '3'):
     # lr = args.learning_rate * (0.2 ** (epoch // 2))
-    if args['lradj'] == 'type1':
+    if lradj == 'type1':
         lr_adjust = {epoch: ['learning_rate'] * (0.5 ** ((epoch - 1) // 1))}
-    elif args['lradj'] == 'type2':
+    elif lradj == 'type2':
         lr_adjust = {
             2: 5e-5, 4: 1e-5, 6: 5e-6, 8: 1e-6,
             10: 5e-7, 15: 1e-7, 20: 5e-8
         }
-    elif args.lradj == 'type3':
-        lr_adjust = {epoch: args.learning_rate if epoch < 3 else args.learning_rate * (0.9 ** ((epoch - 3) // 1))}
-    elif args.lradj == 'constant':
-        lr_adjust = {epoch: args.learning_rate}
-    elif args.lradj == '3':
-        lr_adjust = {epoch: args.learning_rate if epoch < 10 else args.learning_rate * 0.1}
-    elif args.lradj == '4':
-        lr_adjust = {epoch: args.learning_rate if epoch < 15 else args.learning_rate * 0.1}
-    elif args.lradj == '5':
-        lr_adjust = {epoch: args.learning_rate if epoch < 25 else args.learning_rate * 0.1}
-    elif args.lradj == '6':
-        lr_adjust = {epoch: args.learning_rate if epoch < 5 else args.learning_rate * 0.1}
-    elif args.lradj == 'TST':
+    elif lradj == 'type3':
+        lr_adjust = {epoch: learning_rate if epoch < 3 else learning_rate * (0.9 ** ((epoch - 3) // 1))}
+    elif lradj == 'constant':
+        lr_adjust = {epoch:learning_rate}
+    elif lradj == '3':
+        lr_adjust = {epoch: learning_rate if epoch < 10 else learning_rate * 0.1}
+    elif lradj == '4':
+        lr_adjust = {epoch: learning_rate if epoch < 15 else learning_rate * 0.1}
+    elif lradj == '5':
+        lr_adjust = {epoch: learning_rate if epoch < 25 else learning_rate * 0.1}
+    elif lradj == '6':
+        lr_adjust = {epoch: learning_rate if epoch < 5 else learning_rate * 0.1}
+    elif lradj == 'TST':
         lr_adjust = {epoch: scheduler.get_last_lr()[0]}
 
     if epoch in lr_adjust.keys():

fedot_ind/core/operation/interfaces/fedot_nn_evaluation_strategy.py

@@ -38,14 +38,14 @@ def fit(self, train_data: InputData):
         model.fit(train_data)
         return model
 
-    def predict(self, trained_operation, predict_data: InputData) -> OutputData:
-        return trained_operation.predict(predict_data, self.output_mode)
+    def predict(self, trained_operation, predict_data: InputData, output_mode: str = 'default') -> OutputData:
+        return trained_operation.predict(predict_data, output_mode)
 
-    def predict_for_fit(self, trained_operation, predict_data: InputData) -> OutputData:
+    def predict_for_fit(self, trained_operation, predict_data: InputData, output_mode: str = 'default') -> OutputData:
         if 'predict_for_fit' in dir(trained_operation):
-            return trained_operation.predict_for_fit(predict_data, self.output_mode)
+            return trained_operation.predict_for_fit(predict_data, output_mode)
         else:
-            return trained_operation.predict(predict_data, self.output_mode)
+            return trained_operation.predict(predict_data, output_mode)
 
 
 class FedotNNTimeSeriesStrategy(FedotTsForecastingStrategy):

fedot_ind/core/operation/interfaces/industrial_preprocessing_strategy.py

@@ -45,7 +45,8 @@ def __convert_dimensions(self, predict_data, prediction, output_mode: str = 'def
             prediction = prediction.squeeze()
         return prediction
 
-    def _convert_to_output(self, prediction, predict_data: InputData,
+    def _convert_to_output(self, prediction,
+                           predict_data: InputData,
                            output_data_type: DataTypesEnum = DataTypesEnum.table,
                            output_mode: str = 'default') -> OutputData:
         """Method convert prediction into :obj:`OutputData` if it is not this type yet
@@ -57,19 +58,18 @@ def _convert_to_output(self, prediction, predict_data: InputData,
 
         Returns: prediction as :obj:`OutputData`
         """
-        prediction = self.__convert_dimensions(predict_data, prediction, output_mode)
-
-        if type(prediction) is not OutputData:
-            # Wrap prediction as OutputData
+        if type(prediction) is OutputData:
+            prediction.predict = self.__convert_dimensions(predict_data, prediction.predict, output_mode)
+            converted = prediction
+        else:
+            prediction = self.__convert_dimensions(predict_data, prediction, output_mode)
             converted = OutputData(idx=predict_data.idx,
                                    features=predict_data.features,
                                    predict=prediction,
                                    task=predict_data.task,
                                    target=predict_data.target,
                                    data_type=output_data_type,
                                    supplementary_data=predict_data.supplementary_data)
-        else:
-            converted = prediction
 
         return converted
 
@@ -223,6 +223,8 @@ def predict_for_fit(self, trained_operation, predict_data: InputData,
         if mode == 'model_fitting':
             if len(predict_data.features.shape) > 2:
                 converted_predict_data = self._convert_input_data(predict_data, mode='model_fitting')
+            else:
+                converted_predict_data = predict_data
             prediction = self._sklearn_compatible_prediction(trained_operation=trained_operation,
                                                              features=converted_predict_data.features,
                                                              output_mode=output_mode)
@@ -275,6 +277,7 @@ def __init__(self, operation_type: str, params: Optional[OperationParameters] =
         self.operation_impl = self._convert_to_operation(operation_type)
         params = IndustrialOperationParameters().from_params(operation_type, params) if params \
             else IndustrialOperationParameters().from_operation_type(operation_type)
+        self.multi_dim_dispatcher = MultiDimPreprocessingStrategy(self.operation_impl, operation_type)
         super().__init__(operation_type, params)
 
     def _convert_to_operation(self, operation_type: str):
@@ -311,7 +314,7 @@ def predict(self, trained_operation, predict_data: InputData, output_mode: str =
         """
         prediction = trained_operation.transform(predict_data)
         # Convert prediction to output (if it is required)
-        converted = self._convert_to_output(prediction, predict_data)
+        converted = self.multi_dim_dispatcher._convert_to_output(prediction, predict_data)
         return converted
 
     def predict_for_fit(self, trained_operation, predict_data: InputData, output_mode: str = 'default') -> OutputData:
@@ -325,7 +328,7 @@ def predict_for_fit(self, trained_operation, predict_data: InputData, output_mod
             OutputData:
         """
         prediction = trained_operation.transform_for_fit(predict_data)
-        converted = self._convert_to_output(prediction, predict_data)
+        converted = self.multi_dim_dispatcher._convert_to_output(prediction, predict_data)
         return converted