From 450c7dfc5d606448e181bc9d678d8bf7feb55ec5 Mon Sep 17 00:00:00 2001 From: Lopa10ko Date: Thu, 23 Jan 2025 18:23:26 +0300 Subject: [PATCH 1/4] feat: probability calibration new api example --- .../probability_calibration_example.py | 54 +++++++++---------- .../core/repository/config_repository.py | 2 + 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/examples/automl_example/custom_strategy/specific_strategy/probability_calibration_example.py b/examples/automl_example/custom_strategy/specific_strategy/probability_calibration_example.py index 0ce6ab8bc..846d855a4 100644 --- a/examples/automl_example/custom_strategy/specific_strategy/probability_calibration_example.py +++ b/examples/automl_example/custom_strategy/specific_strategy/probability_calibration_example.py @@ -1,42 +1,38 @@ import numpy as np +from examples.example_utils import create_feature_generator_strategy from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate from fedot_ind.core.repository.config_repository import DEFAULT_COMPUTE_CONFIG, \ DEFAULT_AUTOML_LEARNING_CONFIG -INDUSTRIAL_PARAMS = {'data_type': 'tensor', - 'learning_strategy': 'ts2tabular' - } - -# DEFINE ALL CONFIG FOR API -AUTOML_LEARNING_STRATEGY = DEFAULT_AUTOML_LEARNING_CONFIG -COMPUTE_CONFIG = DEFAULT_COMPUTE_CONFIG -AUTOML_CONFIG = {'task': 'classification'} -LEARNING_CONFIG = {'learning_strategy': 'from_scratch', - 'learning_strategy_params': AUTOML_LEARNING_STRATEGY, - 'optimisation_loss': {'quality_loss': 'f1'}} -INDUSTRIAL_CONFIG = {'problem': 'classification', - 'strategy': 'tabular', - 'strategy_params': INDUSTRIAL_PARAMS - } -API_CONFIG = {'industrial_config': INDUSTRIAL_CONFIG, - 'automl_config': AUTOML_CONFIG, - 'learning_config': LEARNING_CONFIG, - 'compute_config': COMPUTE_CONFIG} if __name__ == "__main__": - dataset_name = 'Libras' - finetune = False - api_config = dict(problem='classification', - metric='f1', - timeout=0.1, - n_jobs=2, - logging_level=20) + DATASET_NAME = 'Libras' + + feature_generator, sampling_dict = create_feature_generator_strategy() + DEFAULT_AUTOML_LEARNING_CONFIG['timeout'] = 0.1 + + AUTOML_CONFIG = {'task': 'classification'} + LEARNING_CONFIG = {'learning_strategy': 'from_scratch', + 'learning_strategy_params': DEFAULT_AUTOML_LEARNING_CONFIG, + 'optimisation_loss': {'quality_loss': 'f1'}} + INDUSTRIAL_CONFIG = {'problem': 'classification', + 'strategy': 'tabular', + 'strategy_params': {'feature_generator': feature_generator, + 'data_type': 'tensor', + 'learning_strategy': 'ts2tabular', + 'sampling_strategy': sampling_dict}} + + API_CONFIG = {'industrial_config': INDUSTRIAL_CONFIG, + 'automl_config': AUTOML_CONFIG, + 'learning_config': LEARNING_CONFIG, + 'compute_config': DEFAULT_COMPUTE_CONFIG} + api_client = ApiTemplate(api_config=API_CONFIG, metric_list=('f1', 'accuracy')) - result_dict = api_client.eval(dataset=dataset_name, finetune=finetune) - uncalibrated_labels, uncalibrated_probs = result_dict['industrial_model'].predicted_labels, \ - result_dict['industrial_model'].predicted_probs + result_dict = api_client.eval(dataset=DATASET_NAME, finetune=False) + uncalibrated_labels, uncalibrated_probs = result_dict['industrial_model'].manager.predicted_labels, \ + result_dict['industrial_model'].manager.predicted_probs calibrated_probs = result_dict['industrial_model'].predict_proba(predict_data=api_client.test_data, calibrate_probs=True) calibrated_labels = np.argmax(calibrated_probs, axis=1) + np.min(uncalibrated_labels) diff --git a/fedot_ind/core/repository/config_repository.py b/fedot_ind/core/repository/config_repository.py index 269618923..cb9197e31 100644 --- a/fedot_ind/core/repository/config_repository.py +++ b/fedot_ind/core/repository/config_repository.py @@ -32,9 +32,11 @@ class AutomlConfigConstant(Enum): 'optimisation_agent': 'Industrial'}} DEFAULT_CLF_AUTOML_CONFIG = {'task': 'classification', **DEFAULT_SUBCONFIG} DEFAULT_REG_AUTOML_CONFIG = {'task': 'regression', **DEFAULT_SUBCONFIG} + DEFAULT_TSF_AUTOML_CONFIG = {'task': 'ts_forecasting', **DEFAULT_SUBCONFIG} DEFAULT_AUTOML_LEARNING_CONFIG = AutomlLearningConfigConstant.DEFAULT_AUTOML_CONFIG.value DEFAULT_COMPUTE_CONFIG = ComputeConfigConstant.DEFAULT_COMPUTE_CONFIG.value DEFAULT_CLF_AUTOML_CONFIG = AutomlConfigConstant.DEFAULT_CLF_AUTOML_CONFIG.value DEFAULT_REG_AUTOML_CONFIG = AutomlConfigConstant.DEFAULT_REG_AUTOML_CONFIG.value +DEFAULT_TSF_AUTOML_CONFIG = AutomlConfigConstant.DEFAULT_TSF_AUTOML_CONFIG.value From b748c072d69afbcf5b97056a3c73552f94414bb1 Mon Sep 17 00:00:00 2001 From: Lopa10ko Date: Thu, 23 Jan 2025 18:59:56 +0300 Subject: [PATCH 2/4] feat: forecasting strategy example --- .../forecasting_strategy_example.py | 67 +++++++++---------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py b/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py index ebef7e8c8..055ba58d1 100644 --- a/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py +++ b/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py @@ -1,50 +1,43 @@ import pickle from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate +from fedot_ind.core.repository.config_repository import DEFAULT_COMPUTE_CONFIG, DEFAULT_TSF_AUTOML_CONFIG, \ + DEFAULT_AUTOML_LEARNING_CONFIG from fedot_ind.core.repository.constanst_repository import M4_FORECASTING_BENCH -finetune = False - - -def forecasting_loop(dataset_dict, api_config): - metric_names = ('rmse', 'smape') - result_dict = ApiTemplate(api_config=api_config, - metric_list=metric_names).eval(dataset=dataset_dict, - finetune=finetune) - - return result_dict - - -def evaluate_for_M4(type: str = 'M'): - dataset_list = [data for data in M4_FORECASTING_BENCH if data.__contains__(type)] - return dataset_list - if __name__ == "__main__": - bench = 'M4' - group = 'M' - forecast_params = {'forecast_length': 8} - horizon = forecast_params['forecast_length'] - dataset_list = evaluate_for_M4(group) - api_config = dict( - problem='ts_forecasting', - metric='rmse', - timeout=5, - with_tuning=False, - industrial_strategy='forecasting_assumptions', - industrial_strategy_params={ - 'industrial_task': 'ts_forecasting', - 'data_type': 'time_series'}, - task_params=forecast_params, - logging_level=50) + METRIC_NAMES = ('rmse', 'smape') + HORIZON = 8 + TASK_PARAMS = {'forecast_length': HORIZON} + BENCH = 'M4' + GROUP = 'M' + + DATASET_NAMES = [data for data in M4_FORECASTING_BENCH if data.__contains__(GROUP)] + + DEFAULT_AUTOML_LEARNING_CONFIG['timeout'] = 5 + + API_CONFIG = {'industrial_config': {'problem': 'ts_forecasting', + 'data_type': 'time_series', + 'learning_strategy': 'forecasting_assumptions', + 'task_params': TASK_PARAMS}, + 'automl_config': {**DEFAULT_TSF_AUTOML_CONFIG, + 'task_params': TASK_PARAMS}, + 'learning_config': {'learning_strategy': 'from_scratch', + 'learning_strategy_params': DEFAULT_AUTOML_LEARNING_CONFIG, + 'optimisation_loss': {'quality_loss': 'rmse'}}, + 'compute_config': DEFAULT_COMPUTE_CONFIG} + result_dict = {} - for dataset_name in dataset_list: - dataset_dict = {'benchmark': bench, + for dataset_name in DATASET_NAMES: + dataset_dict = {'benchmark': BENCH, 'dataset': dataset_name, - 'task_params': forecast_params} - result_dict = forecasting_loop(dataset_dict, api_config) + 'task_params': TASK_PARAMS} + result_dict = ApiTemplate(api_config=API_CONFIG, + metric_list=METRIC_NAMES).eval(dataset=dataset_dict, + finetune=False) result_dict.update({dataset_name: result_dict}) - with open(f'{bench}_{group}_forecast_length_{horizon}.pkl', 'wb') as f: + with open(f'{BENCH}_{GROUP}_forecast_length_{HORIZON}.pkl', 'wb') as f: pickle.dump(result_dict, f) From 46802ad402508c1afdfd640d911429a1f951d22d Mon Sep 17 00:00:00 2001 From: autopep8 bot Date: Thu, 23 Jan 2025 16:01:06 +0000 Subject: [PATCH 3/4] Automated autopep8 fixes --- .../ts_forecasting/forecasting_strategy_example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py b/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py index 055ba58d1..7354f4af9 100644 --- a/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py +++ b/examples/automl_example/time_series/ts_forecasting/forecasting_strategy_example.py @@ -35,8 +35,8 @@ 'dataset': dataset_name, 'task_params': TASK_PARAMS} result_dict = ApiTemplate(api_config=API_CONFIG, - metric_list=METRIC_NAMES).eval(dataset=dataset_dict, - finetune=False) + metric_list=METRIC_NAMES).eval(dataset=dataset_dict, + finetune=False) result_dict.update({dataset_name: result_dict}) with open(f'{BENCH}_{GROUP}_forecast_length_{HORIZON}.pkl', 'wb') as f: From 6c4eee5cbc6162f2a60672b006225119cf4efd00 Mon Sep 17 00:00:00 2001 From: Lopa10ko Date: Fri, 24 Jan 2025 14:14:32 +0300 Subject: [PATCH 4/4] fix: pass strategy_params instead of a task_params to a strategy constructor, add exog example --- .../ts_forecasting/ts_forecasting_exogen.py | 54 ++++++++++--------- fedot_ind/api/utils/api_init.py | 2 +- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/examples/automl_example/time_series/ts_forecasting/ts_forecasting_exogen.py b/examples/automl_example/time_series/ts_forecasting/ts_forecasting_exogen.py index 1967a5a8b..1c5527731 100644 --- a/examples/automl_example/time_series/ts_forecasting/ts_forecasting_exogen.py +++ b/examples/automl_example/time_series/ts_forecasting/ts_forecasting_exogen.py @@ -1,35 +1,39 @@ import numpy as np -import pandas as pd +from examples.example_utils import load_monash_dataset from fedot_ind.api.main import FedotIndustrial -from fedot_ind.tools.serialisation.path_lib import PROJECT_PATH +from fedot_ind.core.repository.config_repository import DEFAULT_COMPUTE_CONFIG, DEFAULT_TSF_AUTOML_CONFIG if __name__ == "__main__": - dataset_name = PROJECT_PATH + \ - '/examples/data/forecasting\\monash_benchmark\\MonashBitcoin_30.csv' - horizon = 60 - metric_names = ('smape', 'rmse', 'median_absolute_error') + HORIZON = 60 + METRIC_NAMES = ('smape', 'rmse', 'median_absolute_error') - train_data = pd.read_csv(dataset_name) - variables = train_data['label'].unique().tolist() - exog_var = ['send_usd', 'market_cap', - 'median_transaction_value', 'google_trends'] - exog_ts = np.vstack( - [train_data[train_data['label'] == var]['value'].values for var in exog_var]) + train_data = load_monash_dataset('bitcoin') + exog_var = ['send_usd', 'market_cap', 'median_transaction_value', 'google_trends'] + exog_ts = np.vstack([train_data[column].values for column in exog_var]) exog_ts = exog_ts[0, :] - ts = train_data[train_data['label'] == 'price']['value'].values - target = ts[-horizon:].flatten() + ts = train_data['price'].values + target = ts[-HORIZON:].flatten() input_data = (ts, target) - api_config = dict(problem='ts_forecasting', - metric='rmse', - timeout=15, - with_tuning=False, - pop_size=10, - industrial_strategy_params={'exog_variable': exog_ts}, - task_params={'forecast_length': horizon}, - industrial_strategy='forecasting_exogenous', - n_jobs=2, - logging_level=30) - industrial = FedotIndustrial(**api_config) + TASK_PARAMS = {'forecast_length': HORIZON} + AUTOML_LEARNING_STRATEGY = dict(timeout=3, + with_tuning=False, + n_jobs=2, + pop_size=10, + logging_level=30) + + API_CONFIG = {'industrial_config': {'problem': 'ts_forecasting', + 'task_params': TASK_PARAMS, + 'strategy': 'forecasting_exogenous', + 'strategy_params': {'exog_variable': exog_ts, + 'data_type': 'time_series'}}, + 'automl_config': {'task_params': TASK_PARAMS, + **DEFAULT_TSF_AUTOML_CONFIG}, + 'learning_config': {'learning_strategy': 'from_scratch', + 'learning_strategy_params': AUTOML_LEARNING_STRATEGY, + 'optimisation_loss': {'quality_loss': 'rmse'}}, + 'compute_config': DEFAULT_COMPUTE_CONFIG} + + industrial = FedotIndustrial(**API_CONFIG) industrial.fit(input_data) diff --git a/fedot_ind/api/utils/api_init.py b/fedot_ind/api/utils/api_init.py index fbe3bffb6..1432bf931 100644 --- a/fedot_ind/api/utils/api_init.py +++ b/fedot_ind/api/utils/api_init.py @@ -95,7 +95,7 @@ def build(self, config: dict = None): self.config.update({key: val}) if self.strategy in FEDOT_INDUSTRIAL_STRATEGY: self.strategy = IndustrialStrategy(industrial_strategy=self.strategy, - industrial_strategy_params=self.task_params, + industrial_strategy_params=self.strategy_params, api_config=self.config) return self