Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/industrial_release_0.5' into ind…
Browse files Browse the repository at this point in the history 
…ustrial_release_0.5
  • Loading branch information
@Lopa10ko
Lopa10ko committed Jan 27, 2025
2 parents 8e0c6be + c62b0ce commit c567c3c
Show file tree
Hide file tree
Showing 17 changed files with 442 additions and 338 deletions.
208 changes: 208 additions & 0 deletions examples/real_world_examples/benchmark_example/analysis of results/pdl_uni_benchmark.ipynb
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"collapsed": true,
"pycharm": {
"name": "#%%\n"
}
},
"outputs": [],
"source": [
"from fedot_ind.tools.serialisation.path_lib import PROJECT_PATH\n",
"from fedot_ind.core.repository.constanst_repository import UNI_CLF_BENCH\n",
"import pandas as pd\n",
"from fedot_ind.core.repository.config_repository import DEFAULT_COMPUTE_CONFIG, DEFAULT_CLF_AUTOML_CONFIG\n",
"from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate"
]
},
{
"cell_type": "code",
"execution_count": 2,
"outputs": [],
"source": [
"METRIC_NAMES = ('f1', 'accuracy', 'precision', 'roc_auc')\n",
"COMPUTE_CONFIG = DEFAULT_COMPUTE_CONFIG\n",
"AUTOML_LEARNING_STRATEGY = dict(timeout=2,\n",
" pop_size=10,\n",
" n_jobs=-1,\n",
" num_of_generations=15)\n",
"\n",
"LEARNING_CONFIG = {'learning_strategy': 'from_scratch',\n",
" 'learning_strategy_params': AUTOML_LEARNING_STRATEGY,\n",
" 'optimisation_loss': {'quality_loss': 'accuracy'}}\n",
"\n",
"INDUSTRIAL_CONFIG = {'problem': 'classification'}\n",
"\n",
"API_CONFIG = {'industrial_config': INDUSTRIAL_CONFIG,\n",
" 'automl_config': DEFAULT_CLF_AUTOML_CONFIG,\n",
" 'learning_config': LEARNING_CONFIG,\n",
" 'compute_config': COMPUTE_CONFIG}\n",
"BENCHMARK_PATH = PROJECT_PATH + '/examples/real_world_examples/benchmark_example/classification/UCR_UNI_23_01_25'"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 3,
"outputs": [],
"source": [
"def filter_datasets(UNI_CLF_BENCH, max_classes: int = 10, max_samples: int = 1000):\n",
" UNI_CLF_BENCH_METADATA = pd.read_csv(PROJECT_PATH + '/fedot_ind/core/repository/data/ts_benchmark_metadata.csv')\n",
" datasets_filtred_by_classes = UNI_CLF_BENCH_METADATA[UNI_CLF_BENCH_METADATA['Class'] <= max_classes]\n",
" datasets_filtred_by_samples = datasets_filtred_by_classes[datasets_filtred_by_classes['Train ']\n",
" <= max_samples]\n",
" datasets_filtred_by_samples = datasets_filtred_by_samples[datasets_filtred_by_samples['Test ']\n",
" <= max_samples]['Name'].values.tolist()\n",
" UNI_CLF_BENCH = [x for x in UNI_CLF_BENCH if x in datasets_filtred_by_samples ]\n",
" UNI_CLF_BENCH_METADATA = UNI_CLF_BENCH_METADATA[UNI_CLF_BENCH_METADATA['Name'].isin(datasets_filtred_by_samples)]\n",
" return UNI_CLF_BENCH, UNI_CLF_BENCH_METADATA"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 4,
"outputs": [],
"source": [
"UNI_CLF_BENCH, UNI_CLF_BENCH_METADATA = filter_datasets(UNI_CLF_BENCH)"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 5,
"outputs": [],
"source": [
"api_agent = ApiTemplate(api_config=API_CONFIG, metric_list=METRIC_NAMES)"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 6,
"outputs": [],
"source": [
"bench_results = api_agent.load_result(benchmark_path=BENCHMARK_PATH)"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 7,
"outputs": [],
"source": [
"df_list = list(bench_results.values())"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 8,
"outputs": [],
"source": [
"new_df=pd.merge(\n",
" left=df_list[0],\n",
" right=df_list[1],\n",
" how='left',\n",
" left_on=['dataset'],\n",
" right_on=['dataset'])"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": 9,
"outputs": [
{
"data": {
"text/plain": " accuracy_pdl_rf f1_pdl_rf precision_pdl_rf dataset \\\n0 0.120 0.120 0.120 ACSF1 \n1 0.497 0.509 0.516 ArrowHead \n2 0.167 0.145 0.132 Beef \n3 0.400 0.400 0.400 BeetleFly \n4 0.500 0.000 0.250 BirdChicken \n5 0.280 0.285 0.299 BME \n6 0.317 0.261 0.246 Car \n7 0.226 0.123 0.084 CBF \n8 0.254 0.158 0.294 Chinatown \n9 0.429 0.429 0.431 Coffee \n10 0.432 0.437 0.432 Computers \n11 0.405 0.354 0.263 DiatomSizeReduction \n\n accuracy_rf f1_rf precision_rf \n0 0.120 0.120 0.123 \n1 0.331 0.317 0.312 \n2 0.133 0.124 0.117 \n3 0.450 0.353 0.445 \n4 0.600 0.556 0.604 \n5 0.300 0.288 0.285 \n6 0.267 0.274 0.318 \n7 0.336 0.169 0.112 \n8 0.813 0.714 0.775 \n9 0.929 0.923 0.928 \n10 0.796 0.792 0.796 \n11 0.892 0.844 0.680 ",
"text/html": "<div>\n<style scoped>\n .dataframe tbody tr th:only-of-type {\n vertical-align: middle;\n }\n\n .dataframe tbody tr th {\n vertical-align: top;\n }\n\n .dataframe thead th {\n text-align: right;\n }\n</style>\n<table border=\"1\" class=\"dataframe\">\n <thead>\n <tr style=\"text-align: right;\">\n <th></th>\n <th>accuracy_pdl_rf</th>\n <th>f1_pdl_rf</th>\n <th>precision_pdl_rf</th>\n <th>dataset</th>\n <th>accuracy_rf</th>\n <th>f1_rf</th>\n <th>precision_rf</th>\n </tr>\n </thead>\n <tbody>\n <tr>\n <th>0</th>\n <td>0.120</td>\n <td>0.120</td>\n <td>0.120</td>\n <td>ACSF1</td>\n <td>0.120</td>\n <td>0.120</td>\n <td>0.123</td>\n </tr>\n <tr>\n <th>1</th>\n <td>0.497</td>\n <td>0.509</td>\n <td>0.516</td>\n <td>ArrowHead</td>\n <td>0.331</td>\n <td>0.317</td>\n <td>0.312</td>\n </tr>\n <tr>\n <th>2</th>\n <td>0.167</td>\n <td>0.145</td>\n <td>0.132</td>\n <td>Beef</td>\n <td>0.133</td>\n <td>0.124</td>\n <td>0.117</td>\n </tr>\n <tr>\n <th>3</th>\n <td>0.400</td>\n <td>0.400</td>\n <td>0.400</td>\n <td>BeetleFly</td>\n <td>0.450</td>\n <td>0.353</td>\n <td>0.445</td>\n </tr>\n <tr>\n <th>4</th>\n <td>0.500</td>\n <td>0.000</td>\n <td>0.250</td>\n <td>BirdChicken</td>\n <td>0.600</td>\n <td>0.556</td>\n <td>0.604</td>\n </tr>\n <tr>\n <th>5</th>\n <td>0.280</td>\n <td>0.285</td>\n <td>0.299</td>\n <td>BME</td>\n <td>0.300</td>\n <td>0.288</td>\n <td>0.285</td>\n </tr>\n <tr>\n <th>6</th>\n <td>0.317</td>\n <td>0.261</td>\n <td>0.246</td>\n <td>Car</td>\n <td>0.267</td>\n <td>0.274</td>\n <td>0.318</td>\n </tr>\n <tr>\n <th>7</th>\n <td>0.226</td>\n <td>0.123</td>\n <td>0.084</td>\n <td>CBF</td>\n <td>0.336</td>\n <td>0.169</td>\n <td>0.112</td>\n </tr>\n <tr>\n <th>8</th>\n <td>0.254</td>\n <td>0.158</td>\n <td>0.294</td>\n <td>Chinatown</td>\n <td>0.813</td>\n <td>0.714</td>\n <td>0.775</td>\n </tr>\n <tr>\n <th>9</th>\n <td>0.429</td>\n <td>0.429</td>\n <td>0.431</td>\n <td>Coffee</td>\n <td>0.929</td>\n <td>0.923</td>\n <td>0.928</td>\n </tr>\n <tr>\n <th>10</th>\n <td>0.432</td>\n <td>0.437</td>\n <td>0.432</td>\n <td>Computers</td>\n <td>0.796</td>\n <td>0.792</td>\n <td>0.796</td>\n </tr>\n <tr>\n <th>11</th>\n <td>0.405</td>\n <td>0.354</td>\n <td>0.263</td>\n <td>DiatomSizeReduction</td>\n <td>0.892</td>\n <td>0.844</td>\n <td>0.680</td>\n </tr>\n </tbody>\n</table>\n</div>"
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_df"
],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [],
"metadata": {
"collapsed": false,
"pycharm": {
"name": "#%%\n"
}
}
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 0
}
22 changes: 21 additions & 1 deletion examples/real_world_examples/benchmark_example/classification/PDL_uni.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,34 @@
import os

import pandas as pd

from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate
from fedot_ind.core.repository.config_repository import DEFAULT_COMPUTE_CONFIG, DEFAULT_CLF_AUTOML_CONFIG
from fedot_ind.core.repository.constanst_repository import UNI_CLF_BENCH
from fedot_ind.tools.serialisation.path_lib import PROJECT_PATH


def filter_datasets(UNI_CLF_BENCH, max_classes: int = 10, max_samples: int = 1000):
UNI_CLF_BENCH_METADATA = pd.read_csv(PROJECT_PATH + '/fedot_ind/core/repository/data/ts_benchmark_metadata.csv')
datasets_filtred_by_classes = UNI_CLF_BENCH_METADATA[UNI_CLF_BENCH_METADATA['Class'] <= max_classes]
datasets_filtred_by_samples = datasets_filtred_by_classes[datasets_filtred_by_classes['Train ']
<= max_samples]
datasets_filtred_by_samples = datasets_filtred_by_samples[datasets_filtred_by_samples['Test ']
<= max_samples]['Name'].values.tolist()
already_eval = os.listdir('./UCR_UNI_23_01_25/rf')
UNI_CLF_BENCH = [x for x in UNI_CLF_BENCH if x in datasets_filtred_by_samples and x not in already_eval]
UNI_CLF_BENCH_METADATA = UNI_CLF_BENCH_METADATA[UNI_CLF_BENCH_METADATA['Name'].isin(datasets_filtred_by_samples)]
return UNI_CLF_BENCH, UNI_CLF_BENCH_METADATA


model_to_compare = [{0: ['quantile_extractor', 'rf']},
{0: ['quantile_extractor', 'pdl_clf']}
]
model_name = ['rf', 'pdl_rf']
finutune_existed_model = [True, True]
BENCHMARK = 'UCR_UNI'
BENCHMARK_PARAMS = {'experiment_date': '22_01_25',
UNI_CLF_BENCH, UNI_CLF_BENCH_METADATA = filter_datasets(UNI_CLF_BENCH)
BENCHMARK_PARAMS = {'experiment_date': '23_01_25',
'metadata': None,
'datasets': UNI_CLF_BENCH,
'model_to_compare': (model_to_compare, model_name, finutune_existed_model)}
Expand Down
71 changes: 71 additions & 0 deletions examples/real_world_examples/benchmark_example/detection/ts_anomaly_detection_skab_bench.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import os

from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate
from fedot_ind.core.repository.config_repository import DEFAULT_COMPUTE_CONFIG, DEFAULT_CLF_AUTOML_CONFIG
from tools.test_load_data import EXAMPLES_DATA_PATH


def prepare_skab_benchmark():
ENCODER_LEARNING_PARAMS = {'epochs': 150,
'lr': 0.001,
'device': 'cpu'
}
model_to_compare = [
# {0: ['iforest_detector']},
{0: [('conv_ae_detector', ENCODER_LEARNING_PARAMS)]},
# {0: ['stat_detector']},
# {}
]
model_name = [
# 'iforest',
'conv_encoder',
# 'stat_detector',
# 'industrial'
]
finutune_existed_model = [
True,
True,
# True, False
]
BENCHMARK = 'SKAB'
folder = 'valve1'
datasets = os.listdir(EXAMPLES_DATA_PATH + f'/benchmark/detection/data/{folder}')
datasets = [x.split('.')[0] for x in datasets]
BENCHMARK_PARAMS = {'experiment_date': '23_01_25',
'metadata': {'folder': folder},
'datasets': datasets,
'model_to_compare': (model_to_compare, model_name, finutune_existed_model)}
return BENCHMARK, BENCHMARK_PARAMS


METRIC_NAMES = ('nab', 'accuracy')
EVAL_REGIME = True

COMPUTE_CONFIG = DEFAULT_COMPUTE_CONFIG
AUTOML_CONFIG = DEFAULT_CLF_AUTOML_CONFIG
AUTOML_LEARNING_STRATEGY = dict(timeout=1,
n_jobs=2,
pop_size=10,
logging_level=0)

LEARNING_CONFIG = {'learning_strategy': 'from_scratch',
'learning_strategy_params': AUTOML_LEARNING_STRATEGY,
'optimisation_loss': {'quality_loss': 'accuracy'}}

INDUSTRIAL_CONFIG = {'strategy': 'anomaly_detection',
'problem': 'classification',
'strategy_params': {'detection_window': 10,
'train_data_size': 'anomaly-free',
'data_type': 'time_series'}}

API_CONFIG = {'industrial_config': INDUSTRIAL_CONFIG,
'automl_config': AUTOML_CONFIG,
'learning_config': LEARNING_CONFIG,
'compute_config': COMPUTE_CONFIG}

if __name__ == "__main__":
api_agent = ApiTemplate(api_config=API_CONFIG, metric_list=METRIC_NAMES)
BENCHMARK, BENCHMARK_PARAMS = prepare_skab_benchmark()
if EVAL_REGIME:
api_agent.evaluate_benchmark(benchmark_name=BENCHMARK,
benchmark_params=BENCHMARK_PARAMS)
24 changes: 16 additions & 8 deletions examples/real_world_examples/benchmark_example/forecasting/time_series_uni_forecast_m4.py
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import os

from fedot_ind.core.architecture.pipelines.abstract_pipeline import ApiTemplate
from fedot_ind.core.repository.config_repository import DEFAULT_COMPUTE_CONFIG
from fedot_ind.core.repository.constanst_repository import M4_FORECASTING_BENCH, M4_FORECASTING_LENGTH
Expand All @@ -6,10 +8,6 @@
'lr': 0.001,
'device': 'cpu'
}
# composite = {0: ['lagged', 'ridge'],
# 1: ['ar'],
# 'head': ['bagging']}
# linear = {0:['smoothing','ar']}
model_to_compare = [
{0: ['smoothing', 'lagged', 'ridge']},
{},
Expand All @@ -19,9 +17,9 @@
model_name = ['lagged_regression', 'industrial', 'deepar', 'ar']
finutune_existed_model = [True, False, True, True]
BENCHMARK = 'M4'
EVALUATED = ['D1002', 'D1019', 'D1091', 'D1032']
EVALUATED = []
DATASETS = [x for x in M4_FORECASTING_BENCH if x not in EVALUATED]
BENCHMARK_PARAMS = {'experiment_date': '22_01_25',
BENCHMARK_PARAMS = {'experiment_date': '23_01_25',
'metadata': M4_FORECASTING_LENGTH,
'datasets': DATASETS,
'model_to_compare': (model_to_compare, model_name, finutune_existed_model)}
Expand Down Expand Up @@ -54,5 +52,15 @@
if __name__ == "__main__":
api_agent = ApiTemplate(api_config=API_CONFIG, metric_list=('rmse', 'mae'))
if EVAL_REGIME:
api_agent.evaluate_benchmark(benchmark_name=BENCHMARK,
benchmark_params=BENCHMARK_PARAMS)
for attempt in range(100):
try:
EVALUATED = os.listdir('./M4_23_01_25/ar')
DATASETS = [x for x in M4_FORECASTING_BENCH if x not in EVALUATED]
BENCHMARK_PARAMS = {'experiment_date': '23_01_25',
'metadata': M4_FORECASTING_LENGTH,
'datasets': DATASETS,
'model_to_compare': (model_to_compare, model_name, finutune_existed_model)}
api_agent.evaluate_benchmark(benchmark_name=BENCHMARK,
benchmark_params=BENCHMARK_PARAMS)
except Exception:
print('ERROR')
Loading

0 comments on commit c567c3c

Please sign in to comment.