Dependafix

.github/workflows/poetry_unit_test.yml

@@ -13,7 +13,7 @@ jobs:
     timeout-minutes: 60
     strategy:
       matrix:
-        python-version: [3.9, '3.10']
+        python-version: [3.9, '3.10', '3.11']
 
     steps:
       - uses: actions/checkout@v2

benchmark/benchmark_TSC.py

@@ -9,6 +9,7 @@
 
 from benchmark.abstract_bench import AbstractBenchmark
 from fedot_ind.api.utils.path_lib import PROJECT_PATH
+from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate
 from fedot_ind.core.architecture.postprocessing.results_picker import ResultsPicker
 from fedot_ind.core.architecture.settings.computational import backend_methods as np
 from fedot_ind.core.metrics.metrics_implementation import Accuracy
@@ -27,6 +28,7 @@ def __init__(self,
         self.logger = logging.getLogger(self.__class__.__name__)
 
         self.experiment_setup = experiment_setup
+        self.init_assumption = deepcopy(self.experiment_setup['initial_assumption'])
         self.multi_TSC = MULTI_CLF_BENCH
         self.uni_TSC = UNI_CLF_BENCH
         if custom_datasets is None:
@@ -38,35 +40,50 @@ def __init__(self,
             self.custom_datasets = custom_datasets
 
         if use_small_datasets:
-            self.path_to_result = '/benchmark/results/time_series_uni_clf_comparasion.csv'
-            self.path_to_save = '/benchmark/results/ts_uni_classification'
+            self.path_to_result = 'time_series_uni_clf_comparasion.csv'
+            self.path_to_save = 'ts_uni_classification'
         else:
-            self.path_to_result = '/benchmark/results/time_series_multi_clf_comparasion.csv'
-            self.path_to_save = '/benchmark/results/ts_multi_classification'
-        self.results_picker = ResultsPicker(
-            path=os.path.abspath(self.output_dir))
+            self.path_to_result = 'time_series_multi_clf_comparasion.csv'
+            self.path_to_save = 'ts_multi_classification'
+        self.output_dir = os.path.join(self.experiment_setup['output_folder'], self.path_to_save)
+        self.results_picker = ResultsPicker(path=os.path.abspath(self.output_dir))
+
+    def _run_model_versus_model(self, dataset_name, comparasion_dict):
+        approach_dict = {}
+        for approach in comparasion_dict.keys():
+            result_dict = ApiTemplate(api_config=self.experiment_setup,
+                                      metric_list=self.experiment_setup['metric_names']). \
+                eval(dataset=dataset_name,
+                     initial_assumption=comparasion_dict[approach],
+                     finetune=self.experiment_setup['finetune'])
+            metric = result_dict['metrics'][self.experiment_setup['metric']][0]
+            approach_dict.update({approach: metric})
+        return approach_dict
+
+    def _run_industrial_versus_sota(self, dataset_name):
+        experiment_setup = deepcopy(self.experiment_setup)
+        prediction, target = self.evaluate_loop(dataset_name, experiment_setup)
+        return Accuracy(target, prediction).metric()
 
     def run(self):
         self.logger.info('Benchmark test started')
         basic_results = self.load_local_basic_results()
         metric_dict = {}
         for dataset_name in self.custom_datasets:
-            experiment_setup = deepcopy(self.experiment_setup)
-            prediction, target = self.evaluate_loop(
-                dataset_name, experiment_setup)
-            metric = Accuracy(target, prediction).metric()
-            metric_dict.update({dataset_name: metric})
-            basic_results.loc[dataset_name, 'Fedot_Industrial'] = metric
-            dataset_path = os.path.join(
-                self.experiment_setup['output_folder'],
-                f'{dataset_name}',
-                'metrics_report.csv')
-            basic_results.to_csv(dataset_path)
-            gc.collect()
-        basic_path = os.path.join(
-            self.experiment_setup['output_folder'],
-            'comprasion_metrics_report.csv')
-        basic_results.to_csv(basic_path)
+            try:
+                if isinstance(self.init_assumption, dict):
+                    model_name = list(self.init_assumption.keys())
+                    metric = self._run_model_versus_model(dataset_name, self.init_assumption)
+                else:
+                    metric = self._run_industrial_versus_sota()
+                    model_name = 'Fedot_Industrial'
+                metric_dict.update({dataset_name: metric})
+                basic_results.loc[dataset_name, model_name] = metric
+                if not os.path.exists(self.output_dir):
+                    os.makedirs(self.output_dir)
+                basic_results.to_csv(os.path.join(self.output_dir, self.path_to_result))
+            except Exception:
+                self.logger.info(f"{dataset_name} problem with eval")
         self.logger.info("Benchmark test finished")
 
     def finetune(self):
@@ -128,15 +145,14 @@ def finetune(self):
 
     def load_local_basic_results(self, path: str = None):
         if path is None:
-            path = PROJECT_PATH + self.path_to_result
+            path = os.path.join(self.output_dir, self.path_to_result)
             try:
                 results = pd.read_csv(path, sep=',', index_col=0)
-                results = results.dropna(axis=1, how='all')
-                results = results.dropna(axis=0, how='all')
+                # results = results.fillna()
+                # results = results.dropna(axis=1, how='all')
+                # results = results.dropna(axis=0, how='all')
             except Exception:
                 results = self.load_web_results()
-            self.experiment_setup['output_folder'] = PROJECT_PATH + \
-                self.path_to_save
             return results
         else:
             return self.results_picker.run(get_metrics_df=True, add_info=True)

benchmark/benchmark_TSER.py

@@ -1,19 +1,20 @@
-from fedot_ind.core.repository.constanst_repository import MULTI_REG_BENCH
-from fedot_ind.core.architecture.postprocessing.results_picker import ResultsPicker
-from benchmark.abstract_bench import AbstractBenchmark
-from fedot_ind.core.metrics.metrics_implementation import RMSE
-from fedot_ind.api.utils.path_lib import PROJECT_PATH
-from fedot_ind.api.main import FedotIndustrial
-from fedot.core.pipelines.pipeline import Pipeline
-from fedot.core.pipelines.node import PipelineNode
-import pandas as pd
 import logging
 import os
 from abc import ABC
 from copy import deepcopy
 
 import matplotlib
+import pandas as pd
+from fedot.core.pipelines.node import PipelineNode
+from fedot.core.pipelines.pipeline import Pipeline
 
+from benchmark.abstract_bench import AbstractBenchmark
+from fedot_ind.api.main import FedotIndustrial
+from fedot_ind.api.utils.path_lib import PROJECT_PATH
+from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate
+from fedot_ind.core.architecture.postprocessing.results_picker import ResultsPicker
+from fedot_ind.core.metrics.metrics_implementation import RMSE
+from fedot_ind.core.repository.constanst_repository import MULTI_REG_BENCH
 from fedot_ind.tools.loader import DataLoader
 
 matplotlib.use('TkAgg')
@@ -31,6 +32,7 @@ def __init__(self,
         self.logger = logging.getLogger(self.__class__.__name__)
 
         self.experiment_setup = experiment_setup
+        self.init_assumption = deepcopy(self.experiment_setup['initial_assumption'])
         self.monash_regression = MULTI_REG_BENCH
         if custom_datasets is None:
             self.custom_datasets = self.monash_regression
@@ -40,26 +42,43 @@ def __init__(self,
         self.results_picker = ResultsPicker(
             path=os.path.abspath(self.output_dir))
 
+    def _run_model_versus_model(self, dataset_name, comparasion_dict):
+        approach_dict = {}
+        for approach in comparasion_dict.keys():
+            result_dict = ApiTemplate(api_config=self.experiment_setup,
+                                      metric_list=self.experiment_setup['metric_names']). \
+                eval(dataset=dataset_name,
+                     initial_assumption=comparasion_dict[approach],
+                     finetune=self.experiment_setup['finetune'])
+            metric = result_dict['metrics'][self.experiment_setup['metric']][0]
+            approach_dict.update({approach: metric})
+        return approach_dict
+
+    def _run_industrial_versus_sota(self, dataset_name):
+        experiment_setup = deepcopy(self.experiment_setup)
+        prediction, target = self.evaluate_loop(dataset_name, experiment_setup)
+        return RMSE(target, prediction).metric()
+
     def run(self):
         self.logger.info('Benchmark test started')
         basic_results = self.load_local_basic_results()
         metric_dict = {}
         for dataset_name in self.custom_datasets:
-            experiment_setup = deepcopy(self.experiment_setup)
-            prediction, target = self.evaluate_loop(
-                dataset_name, experiment_setup)
-            metric = RMSE(target, prediction).metric()
-            metric_dict.update({dataset_name: metric})
-            basic_results.loc[dataset_name, 'Fedot_Industrial'] = metric
-            dataset_path = os.path.join(
-                self.experiment_setup['output_folder'],
-                f'{dataset_name}',
-                'metrics_report.csv')
-            basic_results.to_csv(dataset_path)
-        basic_path = os.path.join(
-            self.experiment_setup['output_folder'],
-            'comprasion_metrics_report.csv')
-        basic_results.to_csv(basic_path)
+            try:
+                if isinstance(self.init_assumption, dict):
+                    model_name = list(self.init_assumption.keys())
+                    metric = self._run_model_versus_model(dataset_name, self.init_assumption)
+                else:
+                    metric = self._run_industrial_versus_sota(dataset_name)
+                    model_name = 'Fedot_Industrial'
+                metric_dict.update({dataset_name: metric})
+                basic_results.loc[dataset_name, model_name] = metric
+                basic_path = os.path.join(self.experiment_setup['output_folder'])
+                if not os.path.exists(basic_path):
+                    os.makedirs(basic_path)
+                basic_results.to_csv(os.path.join(basic_path, 'comprasion_metrics_report.csv'))
+            except Exception:
+                self.logger.info(f"{dataset_name} problem with eval")
         self.logger.info("Benchmark test finished")
 
     def load_local_basic_results(self, path: str = None):

examples/automl_example/api_example/computer_vision/image_classification/image_clf_example.py → examples/automl_example/computer_vision/image_classification/image_clf_example.py

@@ -1,4 +1,5 @@
 import random
+
 import matplotlib.pyplot as plt
 from torchvision.transforms import ToTensor, Resize, Compose
 

examples/automl_example/api_example/computer_vision/image_classification/mnist_lora_example.py → examples/automl_example/computer_vision/image_classification/mnist_lora_example.py

@@ -1,14 +1,11 @@
-from fedot_ind.core.models.nn.network_modules.layers.lora import linear_layer_parameterization
-
 import torch
 import torch.nn as nn
 import torch.nn.utils.parametrize as parametrize
-
 import torchvision.datasets as datasets
 import torchvision.transforms as transforms
-
 from tqdm import tqdm
 
+from fedot_ind.core.models.nn.network_modules.layers.lora import linear_layer_parameterization
 
 # Make torch deterministic
 _ = torch.manual_seed(228)

examples/automl_example/api_example/computer_vision/object_detection/obj_rec_example.py → examples/automl_example/computer_vision/object_detection/obj_rec_example.py

@@ -1,9 +1,10 @@
 import os
 import random
+
 import yaml
+from fedot_ind.core.architecture.datasets.visualization import draw_sample_with_bboxes
 
 from fedot_ind.api.main import FedotIndustrial
-from fedot_ind.core.architecture.datasets.visualization import draw_sample_with_bboxes
 
 DATASETS_PATH = os.path.abspath('Warp-D')
 TEST_IMAGE_FOLDER = 'Land-Use_Scene_Classification/images_train_test_val/test'

examples/automl_example/custom_strategy/big_data/big_dataset_utils.py

@@ -0,0 +1,13 @@
+import numpy as np
+
+
+def create_big_dataset():
+    train_X, test_X = np.load(
+        'big_dataset/train_airlinescodrnaadult_fold0.npy'), np.load(
+        'big_dataset/test_airlinescodrnaadult_fold0.npy')
+    train_y, test_y = np.load(
+        'big_dataset/trainy_airlinescodrnaadult_fold0.npy'), np.load(
+        'big_dataset/testy_airlinescodrnaadult_fold0.npy')
+    dataset_dict = dict(train_data=(train_X, train_y),
+                        test_data=(test_X, test_y))
+    return dataset_dict

examples/automl_example/custom_strategy/big_data/random_sampling_example.py

@@ -0,0 +1,36 @@
+from examples.automl_example.custom_strategy.big_data.big_dataset_utils import create_big_dataset
+from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate
+
+cur_params = {'rank': None}
+sampling_algorithm = {'CUR': cur_params}
+
+if __name__ == "__main__":
+    dataset_dict = create_big_dataset()
+    finetune = False
+    metric_names = ('f1', 'accuracy')
+    api_config = dict(problem='classification',
+                      metric='f1',
+                      timeout=40,
+                      pop_size=10,
+                      early_stopping_iterations=10,
+                      early_stopping_timeout=30,
+                      optimizer_params={'mutation_agent': 'bandit',
+                                        'mutation_strategy': 'growth_mutation_strategy'},
+                      with_tunig=False,
+                      preset='classification_tabular',
+                      industrial_strategy_params={'data_type': 'tensor',
+                                                  'learning_strategy': 'big_dataset',
+                                                  'sampling_strategy': sampling_algorithm
+                                                  },
+                      n_jobs=-1,
+                      logging_level=20)
+
+    result_dict = ApiTemplate(api_config=api_config,
+                              metric_list=metric_names).eval(dataset=dataset_dict,
+                                                             finetune=finetune)
+    metrics = result_dict['metrics']
+    metrics.to_csv('./metrics.csv')
+    hist = result_dict['industrial_model'].save_optimization_history(return_history=True)
+    result_dict['industrial_model'].vis_optimisation_history(hist)
+    result_dict['industrial_model'].save_best_model()
+    _ = 1

examples/automl_example/api_example/advanced_example/specific_strategy/LoRa_example.py → examples/automl_example/custom_strategy/specific_strategy/LoRa_example.py

@@ -1,7 +1,8 @@
-from fedot_ind.api.main import FedotIndustrial
 import torchvision.datasets as datasets
 import torchvision.transforms as transforms
 
+from fedot_ind.api.main import FedotIndustrial
+
 transform = transforms.Compose([
     transforms.ToTensor(),
     transforms.Normalize((0.1307,), (0.3081,))

examples/automl_example/api_example/time_series/ts_anomaly_detection/custom_liman_example.py → examples/automl_example/time_series/ts_anomaly_detection/custom_liman_example.py

@@ -8,7 +8,7 @@
 from fedot_ind.api.utils.checkers_collections import DataCheck
 from fedot_ind.api.utils.path_lib import PROJECT_PATH
 from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate
-from fedot_ind.core.models.quantile.quantile_extractor import QuantileExtractor
+from fedot_ind.core.operation.transformation.representation.statistical.quantile_extractor import QuantileExtractor
 from fedot_ind.core.repository.constanst_repository import FEDOT_TASK
 from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels