hot fix after release (#115)

* hot fix after release
aimclub · Feb 6, 2024 · ca1674d · ca1674d
1 parent 9548d29
commit ca1674d
Show file tree

Hide file tree

Showing 82 changed files with 610 additions and 398 deletions.
diff --git a/benchmark/abstract_bench.py b/benchmark/abstract_bench.py
@@ -57,7 +57,8 @@ def run(self):
     def evaluate_loop(self, dataset, experiment_setup: dict = None):
         matplotlib.use('TkAgg')
         train_data, test_data = DataLoader(dataset_name=dataset).load_data()
-        experiment_setup['output_folder'] = experiment_setup['output_folder'] + f'/{dataset}'
+        experiment_setup['output_folder'] = experiment_setup['output_folder'] + \
+            f'/{dataset}'
         model = FedotIndustrial(**experiment_setup)
         model.fit(train_data)
         prediction = model.predict(test_data)
@@ -71,11 +72,13 @@ def evaluate_loop(self, dataset, experiment_setup: dict = None):
 
     def finetune_loop(self, dataset, experiment_setup: dict = None):
         train_data, test_data = DataLoader(dataset_name=dataset).load_data()
-        experiment_setup['output_folder'] = experiment_setup['output_folder'] + f'/{dataset}'
+        experiment_setup['output_folder'] = experiment_setup['output_folder'] + \
+            f'/{dataset}'
         tuning_params = experiment_setup['tuning_params']
         del experiment_setup['tuning_params']
         model = FedotIndustrial(**experiment_setup)
-        model.load(path=experiment_setup['output_folder'] + '/0_pipeline_saved')
+        model.load(
+            path=experiment_setup['output_folder'] + '/0_pipeline_saved')
         model.finetune(train_data, tuning_params=tuning_params)
         prediction = model.finetune_predict(test_data)
         return prediction, model.predict_data.target

diff --git a/benchmark/benchmark_TSC.py b/benchmark/benchmark_TSC.py
@@ -43,40 +43,47 @@ def __init__(self,
         else:
             self.path_to_result = '/benchmark/results/time_series_multi_clf_comparasion.csv'
             self.path_to_save = '/benchmark/results/ts_multi_classification'
-        self.results_picker = ResultsPicker(path=os.path.abspath(self.output_dir))
+        self.results_picker = ResultsPicker(
+            path=os.path.abspath(self.output_dir))
 
     def run(self):
         self.logger.info('Benchmark test started')
         basic_results = self.load_local_basic_results()
         metric_dict = {}
         for dataset_name in self.custom_datasets:
             experiment_setup = deepcopy(self.experiment_setup)
-            prediction, target = self.evaluate_loop(dataset_name, experiment_setup)
+            prediction, target = self.evaluate_loop(
+                dataset_name, experiment_setup)
             metric = Accuracy(target, prediction).metric()
             metric_dict.update({dataset_name: metric})
             basic_results.loc[dataset_name, 'Fedot_Industrial'] = metric
             dataset_path = os.path.join(self.experiment_setup['output_folder'], f'{dataset_name}',
                                         'metrics_report.csv')
             basic_results.to_csv(dataset_path)
             gc.collect()
-        basic_path = os.path.join(self.experiment_setup['output_folder'], 'comprasion_metrics_report.csv')
+        basic_path = os.path.join(
+            self.experiment_setup['output_folder'], 'comprasion_metrics_report.csv')
         basic_results.to_csv(basic_path)
         self.logger.info("Benchmark test finished")
 
     def finetune(self):
         self.logger.info('Benchmark finetune started')
         for dataset_name in self.custom_datasets:
             try:
-                composed_model_path = PROJECT_PATH + self.path_to_save + f'/{dataset_name}' + '/0_pipeline_saved'
+                composed_model_path = PROJECT_PATH + self.path_to_save + \
+                    f'/{dataset_name}' + '/0_pipeline_saved'
                 if os.path.isdir(composed_model_path):
-                    self.experiment_setup['output_folder'] = PROJECT_PATH + self.path_to_save
+                    self.experiment_setup['output_folder'] = PROJECT_PATH + \
+                        self.path_to_save
                     experiment_setup = deepcopy(self.experiment_setup)
-                    prediction, target = self.finetune_loop(dataset_name, experiment_setup)
+                    prediction, target = self.finetune_loop(
+                        dataset_name, experiment_setup)
                     metric = Accuracy(target, prediction).metric()
                     dataset_path = os.path.join(self.experiment_setup['output_folder'], f'{dataset_name}',
                                                 'metrics_report.csv')
                     fedot_results = pd.read_csv(dataset_path, index_col=0)
-                    fedot_results.loc[dataset_name, 'Fedot_Industrial_finetuned'] = metric
+                    fedot_results.loc[dataset_name,
+                                      'Fedot_Industrial_finetuned'] = metric
 
                     fedot_results.to_csv(dataset_path)
                 else:
@@ -95,7 +102,8 @@ def load_local_basic_results(self, path: str = None):
                 results = results.dropna(axis=0, how='all')
             except Exception:
                 results = self.load_web_results()
-            self.experiment_setup['output_folder'] = PROJECT_PATH + self.path_to_save
+            self.experiment_setup['output_folder'] = PROJECT_PATH + \
+                self.path_to_save
             return results
         else:
             return self.results_picker.run(get_metrics_df=True, add_info=True)
@@ -104,13 +112,15 @@ def create_report(self):
         _ = []
         names = []
         for dataset_name in self.custom_datasets:
-            model_result_path = PROJECT_PATH + self.path_to_save + f'/{dataset_name}' + '/metrics_report.csv'
+            model_result_path = PROJECT_PATH + self.path_to_save + \
+                f'/{dataset_name}' + '/metrics_report.csv'
             if os.path.isfile(model_result_path):
                 df = pd.read_csv(model_result_path, index_col=0, sep=',')
                 df = df.fillna(0)
                 if 'Fedot_Industrial_finetuned' not in df.columns:
                     df['Fedot_Industrial_finetuned'] = 0
-                metrics = df.loc[dataset_name, 'Fedot_Industrial':'Fedot_Industrial_finetuned']
+                metrics = df.loc[dataset_name,
+                                 'Fedot_Industrial':'Fedot_Industrial_finetuned']
                 _.append(metrics.T.values)
                 names.append(dataset_name)
         stacked_resutls = np.stack(_, axis=1).T
@@ -123,6 +133,7 @@ def create_report(self):
 
     def load_web_results(self):
         sota_estimators = get_available_estimators()
-        sota_results = get_estimator_results(estimators=sota_estimators['classification'].values.tolist())
+        sota_results = get_estimator_results(
+            estimators=sota_estimators['classification'].values.tolist())
         sota_results_df = pd.DataFrame(sota_results)
         return sota_results_df
diff --git a/benchmark/benchmark_TSER.py b/benchmark/benchmark_TSER.py
@@ -1,3 +1,12 @@
+from fedot_ind.core.repository.constanst_repository import MULTI_REG_BENCH
+from fedot_ind.core.architecture.postprocessing.results_picker import ResultsPicker
+from benchmark.abstract_bench import AbstractBenchmark
+from fedot_ind.core.metrics.metrics_implementation import RMSE
+from fedot_ind.api.utils.path_lib import PROJECT_PATH
+from fedot_ind.api.main import FedotIndustrial
+from fedot.core.pipelines.pipeline import Pipeline
+from fedot.core.pipelines.node import PipelineNode
+import pandas as pd
 import logging
 import os
 from abc import ABC
@@ -8,16 +17,6 @@
 from fedot_ind.tools.loader import DataLoader
 
 matplotlib.use('TkAgg')
-import pandas as pd
-from fedot.core.pipelines.node import PipelineNode
-from fedot.core.pipelines.pipeline import Pipeline
-
-from fedot_ind.api.main import FedotIndustrial
-from fedot_ind.api.utils.path_lib import PROJECT_PATH
-from fedot_ind.core.metrics.metrics_implementation import RMSE
-from benchmark.abstract_bench import AbstractBenchmark
-from fedot_ind.core.architecture.postprocessing.results_picker import ResultsPicker
-from fedot_ind.core.repository.constanst_repository import MULTI_REG_BENCH
 
 
 class BenchmarkTSER(AbstractBenchmark, ABC):
@@ -38,21 +37,25 @@ def __init__(self,
         else:
             self.custom_datasets = custom_datasets
         self.use_small_datasets = use_small_datasets
-        self.results_picker = ResultsPicker(path=os.path.abspath(self.output_dir))
+        self.results_picker = ResultsPicker(
+            path=os.path.abspath(self.output_dir))
 
     def run(self):
         self.logger.info('Benchmark test started')
         basic_results = self.load_local_basic_results()
         metric_dict = {}
         for dataset_name in self.custom_datasets:
             experiment_setup = deepcopy(self.experiment_setup)
-            prediction, target = self.evaluate_loop(dataset_name, experiment_setup)
+            prediction, target = self.evaluate_loop(
+                dataset_name, experiment_setup)
             metric = RMSE(target, prediction).metric()
             metric_dict.update({dataset_name: metric})
             basic_results.loc[dataset_name, 'Fedot_Industrial'] = metric
-            dataset_path = os.path.join(self.experiment_setup['output_folder'], f'{dataset_name}', 'metrics_report.csv')
+            dataset_path = os.path.join(
+                self.experiment_setup['output_folder'], f'{dataset_name}', 'metrics_report.csv')
             basic_results.to_csv(dataset_path)
-        basic_path = os.path.join(self.experiment_setup['output_folder'], 'comprasion_metrics_report.csv')
+        basic_path = os.path.join(
+            self.experiment_setup['output_folder'], 'comprasion_metrics_report.csv')
         basic_results.to_csv(basic_path)
         self.logger.info("Benchmark test finished")
 
@@ -62,22 +65,27 @@ def load_local_basic_results(self, path: str = None):
             results = pd.read_csv(path, sep=';', index_col=0)
             results = results.dropna(axis=1, how='all')
             results = results.dropna(axis=0, how='all')
-            self.experiment_setup['output_folder'] = PROJECT_PATH + '/benchmark/results/ts_regression'
+            self.experiment_setup['output_folder'] = PROJECT_PATH + \
+                '/benchmark/results/ts_regression'
             return results
         else:
             return self.results_picker.run(get_metrics_df=True, add_info=True)
 
     def finetune(self):
         for dataset_name in self.custom_datasets:
             experiment_setup = deepcopy(self.experiment_setup)
-            composed_model_path = PROJECT_PATH + '/benchmark/results/ts_regression' + f'/{dataset_name}' + '/0_pipeline_saved'
+            composed_model_path = PROJECT_PATH + '/benchmark/results/ts_regression' + \
+                f'/{dataset_name}' + '/0_pipeline_saved'
             experiment_setup['output_folder'] = composed_model_path
             del experiment_setup['industrial_preprocessing']
-            prediction, model = self.finetune_loop(dataset_name, experiment_setup)
+            prediction, model = self.finetune_loop(
+                dataset_name, experiment_setup)
             metric = RMSE(model.predict_data.target, prediction).metric()
-            metric_path = PROJECT_PATH + '/benchmark/results/ts_regression' + f'/{dataset_name}' + '/metrics_report.csv'
+            metric_path = PROJECT_PATH + '/benchmark/results/ts_regression' + \
+                f'/{dataset_name}' + '/metrics_report.csv'
             fedot_results = pd.read_csv(metric_path, index_col=0)
-            fedot_results.loc[dataset_name, 'Fedot_Industrial_finetuned'] = metric
+            fedot_results.loc[dataset_name,
+                              'Fedot_Industrial_finetuned'] = metric
             fedot_results.to_csv(metric_path)
             model.save_best_model()
         self.logger.info("Benchmark finetune finished")
@@ -92,14 +100,16 @@ def finetune_loop(self, dataset, experiment_setup: dict = None):
 
     def show_composite_pipeline(self):
         for dataset_name in self.custom_datasets:
-            composed_model_path = PROJECT_PATH + '/benchmark/results/ts_regression' + f'/{dataset_name}' + '/0_pipeline_saved'
+            composed_model_path = PROJECT_PATH + '/benchmark/results/ts_regression' + \
+                f'/{dataset_name}' + '/0_pipeline_saved'
             experiment_setup = deepcopy(self.experiment_setup)
             experiment_setup['output_folder'] = composed_model_path
             del experiment_setup['industrial_preprocessing']
             model = FedotIndustrial(**experiment_setup)
             model.load(path=composed_model_path)
             batch_pipelines = [automl_branch.fitted_operation.model.current_pipeline
                                for automl_branch in model.current_pipeline.nodes if automl_branch.name == 'fedot_regr']
-            pr = PipelineNode('ridge', nodes_from=[p.root_node for p in batch_pipelines])
+            pr = PipelineNode('ridge', nodes_from=[
+                              p.root_node for p in batch_pipelines])
             composed_pipeline = Pipeline(pr)
             composed_pipeline.show()
diff --git a/benchmark/benchmark_TSF.py b/benchmark/benchmark_TSF.py
@@ -47,12 +47,15 @@ def __init__(self,
         else:
             self.path_to_result = '/benchmark/results/time_series_multi_forecast_comparasion.csv'
             self.path_to_save = '/benchmark/results/ts_multi_forecasting'
-        self.results_picker = ResultsPicker(path=os.path.abspath(self.output_dir))
+        self.results_picker = ResultsPicker(
+            path=os.path.abspath(self.output_dir))
 
     def evaluate_loop(self, dataset, experiment_setup: dict = None):
         matplotlib.use('TkAgg')
-        train_data, test_data = DataLoader(dataset_name=dataset).load_forecast_data()
-        experiment_setup['output_folder'] = experiment_setup['output_folder'] + f'/{dataset}'
+        train_data, test_data = DataLoader(
+            dataset_name=dataset).load_forecast_data()
+        experiment_setup['output_folder'] = experiment_setup['output_folder'] + \
+            f'/{dataset}'
         model = FedotIndustrial(**experiment_setup)
         model.fit(train_data)
         prediction = model.predict(test_data)
@@ -71,34 +74,40 @@ def run(self):
         for dataset_name in self.custom_datasets:
             try:
                 experiment_setup = deepcopy(self.experiment_setup)
-                prediction, target = self.evaluate_loop(dataset_name, experiment_setup)
+                prediction, target = self.evaluate_loop(
+                    dataset_name, experiment_setup)
                 metric = RMSE(target, prediction).metric()
                 metric_dict.update({dataset_name: metric})
                 basic_results.loc[dataset_name, 'Fedot_Industrial'] = metric
                 dataset_path = os.path.join(self.experiment_setup['output_folder'], f'{dataset_name}',
                                             'metrics_report.csv')
                 basic_results.to_csv(dataset_path)
             except Exception as ex:
-                 print(f'Skip dataset.{ex}')
+                print(f'Skip dataset.{ex}')
             gc.collect()
-        basic_path = os.path.join(self.experiment_setup['output_folder'], 'comprasion_metrics_report.csv')
+        basic_path = os.path.join(
+            self.experiment_setup['output_folder'], 'comprasion_metrics_report.csv')
         basic_results.to_csv(basic_path)
         self.logger.info("Benchmark test finished")
 
     def finetune(self):
         self.logger.info('Benchmark finetune started')
         for dataset_name in self.custom_datasets:
             try:
-                composed_model_path = PROJECT_PATH + self.path_to_save + f'/{dataset_name}' + '/0_pipeline_saved'
+                composed_model_path = PROJECT_PATH + self.path_to_save + \
+                    f'/{dataset_name}' + '/0_pipeline_saved'
                 if os.path.isdir(composed_model_path):
-                    self.experiment_setup['output_folder'] = PROJECT_PATH + self.path_to_save
+                    self.experiment_setup['output_folder'] = PROJECT_PATH + \
+                        self.path_to_save
                     experiment_setup = deepcopy(self.experiment_setup)
-                    prediction, target = self.finetune_loop(dataset_name, experiment_setup)
+                    prediction, target = self.finetune_loop(
+                        dataset_name, experiment_setup)
                     metric = Accuracy(target, prediction).metric()
                     dataset_path = os.path.join(self.experiment_setup['output_folder'], f'{dataset_name}',
                                                 'metrics_report.csv')
                     fedot_results = pd.read_csv(dataset_path, index_col=0)
-                    fedot_results.loc[dataset_name, 'Fedot_Industrial_finetuned'] = metric
+                    fedot_results.loc[dataset_name,
+                                      'Fedot_Industrial_finetuned'] = metric
 
                     fedot_results.to_csv(dataset_path)
                 else:
@@ -117,7 +126,8 @@ def load_local_basic_results(self, path: str = None):
                 results = results.dropna(axis=0, how='all')
             except Exception:
                 results = self.load_web_results()
-            self.experiment_setup['output_folder'] = PROJECT_PATH + self.path_to_save
+            self.experiment_setup['output_folder'] = PROJECT_PATH + \
+                self.path_to_save
             return results
         else:
             return self.results_picker.run(get_metrics_df=True, add_info=True)
@@ -126,13 +136,15 @@ def create_report(self):
         _ = []
         names = []
         for dataset_name in self.custom_datasets:
-            model_result_path = PROJECT_PATH + self.path_to_save + f'/{dataset_name}' + '/metrics_report.csv'
+            model_result_path = PROJECT_PATH + self.path_to_save + \
+                f'/{dataset_name}' + '/metrics_report.csv'
             if os.path.isfile(model_result_path):
                 df = pd.read_csv(model_result_path, index_col=0, sep=',')
                 df = df.fillna(0)
                 if 'Fedot_Industrial_finetuned' not in df.columns:
                     df['Fedot_Industrial_finetuned'] = 0
-                metrics = df.loc[dataset_name, 'Fedot_Industrial':'Fedot_Industrial_finetuned']
+                metrics = df.loc[dataset_name,
+                                 'Fedot_Industrial':'Fedot_Industrial_finetuned']
                 _.append(metrics.T.values)
                 names.append(dataset_name)
         stacked_resutls = np.stack(_, axis=1).T
@@ -145,6 +157,7 @@ def create_report(self):
 
     def load_web_results(self):
         sota_estimators = get_available_estimators()
-        sota_results = get_estimator_results(estimators=sota_estimators['classification'].values.tolist())
+        sota_results = get_estimator_results(
+            estimators=sota_estimators['classification'].values.tolist())
         sota_results_df = pd.DataFrame(sota_results)
         return sota_results_df
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -10,11 +10,11 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
+from sphinx.builders.html import StandaloneHTMLBuilder
 import os
 import datetime
 import sys
 sys.path.insert(0, os.path.abspath('../..'))
-from sphinx.builders.html import StandaloneHTMLBuilder
 
 # -- Project information -----------------------------------------------------
 
@@ -105,5 +105,3 @@
 autodoc_typehints = 'signature'
 autodoc_typehints_format = 'short'
 autodoc_mock_imports = ['objgraph', 'memory_profiler', 'gprof2dot', 'snakeviz']
-
-
diff --git a/...ples/automl_example/api_example/computer_vision/image_classification/image_clf_example.py b/...ples/automl_example/api_example/computer_vision/image_classification/image_clf_example.py
@@ -14,6 +14,7 @@
               'advanced': FedotIndustrial(task=TASK, num_classes=NUM_CLASSES, optimization='svd',
                                           optimization_params={'energy_thresholds': [0.99]})}
 
+
 def run_industrial_model(model_type: str = 'basic'):
     fed = model_dict[model_type]