Skip to content

Commit

Permalink
add benchmarks. fix issue with custom model
Browse files Browse the repository at this point in the history
  • Loading branch information
v1docq committed Dec 15, 2023
1 parent 2a32423 commit 054c0b4
Show file tree
Hide file tree
Showing 16 changed files with 419 additions and 205 deletions.
27 changes: 27 additions & 0 deletions examples/benchmark/time_series_multI_comparasion.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
;Arsenal;CBOSS;CIF;DrCIF;DTW_A;DTW_D;DTW_I;gRSF;HC1;HC2;InceptionTime;mrseql;MUSE;ResNet;RISE;ROCKET;STC;TapNet;TDE;TSF;Fedot_Ind
ArticularyWordRecognition;0.9933333333333333;0.99;0.9833333333333333;0.98;0.9866666666666667;0.9866666666666667;0.9533333333333334;0.9833333333333333;0.99;0.9933333333333333;0.9833333333333333;0.9933333333333333;0.9933333333333333;0.98;0.9633333333333334;0.9933333333333333;0.99;0.9566666666666667;0.9933333333333333;0.9533333333333334;0.9766666666666667
AtrialFibrillation;0.13333333333333333;0.26666666666666666;0.3333333333333333;0.3333333333333333;0.26666666666666666;0.2;0.26666666666666666;0.26666666666666666;0.13333333333333333;0.26666666666666666;0.2;0.26666666666666666;0.4;0.3333333333333333;0.26666666666666666;0.06666666666666667;0.26666666666666666;0.2;0.26666666666666666;0.2;0.26666666666666666
BasicMotions;1.0;1.0;1.0;1.0;1.0;0.975;0.725;1.0;1.0;1.0;1.0;0.925;1.0;1.0;1.0;1.0;0.975;1.0;1.0;1.0;1.0
Cricket;1.0;0.9861111111111112;0.9861111111111112;0.9861111111111112;1.0;1.0;0.9583333333333334;0.9861111111111112;0.9861111111111112;1.0;0.9861111111111112;0.9861111111111112;0.9861111111111112;0.9861111111111112;0.9861111111111112;1.0;0.9861111111111112;1.0;0.9861111111111112;0.9305555555555556;1.0
DuckDuckGeese;0.46;0.38;0.44;0.54;0.5;0.58;0.3;0.4;0.48;0.56;;;;0.62;0.46;0.52;0.36;;0.34;0.22;0.0
EigenWorms;0.9007633587786259;0.6183206106870229;0.916030534351145;0.9236641221374046;;0.6183206106870229;0.5954198473282443;0.816793893129771;0.6335877862595419;0.9465648854961832;;;;0.17557251908396945;0.816793893129771;0.9083969465648855;0.7786259541984732;;0.9389312977099237;0.7404580152671756;0.0
Epilepsy;0.9855072463768116;1.0;0.9855072463768116;0.9782608695652174;0.9782608695652174;0.9637681159420289;0.6304347826086957;0.9782608695652174;1.0;1.0;0.9855072463768116;1.0;0.9927536231884058;0.9927536231884058;1.0;0.9927536231884058;0.9927536231884058;0.9565217391304348;0.9927536231884058;0.9782608695652174;0.0
EthanolConcentration;0.4600760456273764;0.3612167300380228;0.7338403041825095;0.6920152091254753;0.3155893536121673;0.3231939163498099;0.3041825095057034;0.34600760456273766;0.7908745247148289;0.7718631178707225;0.23193916349809887;0.5741444866920152;0.4752851711026616;0.2585551330798479;0.4866920152091255;0.4448669201520912;0.8212927756653993;0.30798479087452474;0.5551330798479087;0.4448669201520912;0.2813688212927757
ERing;0.9814814814814815;0.9074074074074074;0.9814814814814815;0.9925925925925926;0.9259259259259259;0.9148148148148149;0.9111111111111111;0.9518518518518518;0.9703703703703703;0.9888888888888889;0.8777777777777778;0.9444444444444444;0.9740740740740741;0.8555555555555555;0.8592592592592593;0.9851851851851852;0.8888888888888888;0.9037037037037037;0.9629629629629629;0.8814814814814815;0.9259259259259259
FaceDetection;0.6532349602724177;0.5158910329171397;0.6271282633371169;0.6200340522133939;0.5283768444948922;0.5286606129398411;0.5158910329171397;0.5476730987514189;0.6563564131668559;0.6603291713961408;;;;0.5876844494892168;0.5079455164585698;0.6438706015891033;0.6461407491486947;;0.5644154370034052;0.6396140749148694;0.0
FingerMovements;0.53;0.48;0.52;0.6;0.51;0.53;0.51;0.58;0.55;0.53;0.56;0.57;0.55;0.57;0.56;0.54;0.51;0.47;0.56;0.58;0.59
HandMovementDirection;0.47297297297297297;0.1891891891891892;0.5945945945945946;0.527027027027027;0.20270270270270271;0.1891891891891892;0.1891891891891892;0.4189189189189189;0.44594594594594594;0.47297297297297297;0.36486486486486486;0.2702702702702703;0.36486486486486486;0.24324324324324326;0.2972972972972973;0.5135135135135135;0.3918918918918919;0.33783783783783783;0.3783783783783784;0.4864864864864865;0.5405405405405406
Handwriting;0.5411764705882353;0.4717647058823529;0.3564705882352941;0.3458823529411765;0.6070588235294118;0.6070588235294118;0.3764705882352941;0.37529411764705883;0.4823529411764706;0.548235294117647;0.6423529411764706;0.5329411764705883;0.5223529411764706;0.5905882352941176;0.19411764705882353;0.5952941176470589;0.28823529411764703;0.2811764705882353;0.5611764705882353;0.3658823529411765;0.0
Heartbeat;0.7414634146341463;0.7219512195121951;0.7804878048780488;0.7902439024390244;0.6926829268292682;0.7170731707317073;0.6146341463414634;0.7609756097560976;0.7219512195121951;0.7317073170731707;0.5804878048780487;0.7365853658536585;0.7121951219512195;0.6878048780487804;0.7317073170731707;0.7463414634146341;0.7219512195121951;0.7902439024390244;0.7463414634146341;0.7414634146341463;0.0
Libras;0.9055555555555556;0.8444444444444444;0.9111111111111111;0.8944444444444445;0.8833333333333333;0.8722222222222222;0.8333333333333334;0.6944444444444444;0.9;0.9333333333333333;0.8888888888888888;0.8888888888888888;0.8944444444444445;0.9444444444444444;0.8055555555555556;0.9;0.8611111111111112;0.8777777777777778;0.85;0.8055555555555556;0.7111111111111111
LSST;0.6423357664233577;0.43471208434712083;0.5725871857258719;0.5559610705596107;0.5673154906731549;0.551094890510949;0.48986212489862124;0.5884022708840227;0.5754257907542579;0.6427412814274128;0.6123276561232766;0.6119221411192214;0.6399026763990268;0.6196269261962692;0.5089213300892133;0.6366585563665855;0.5867802108678021;0.5125709651257097;0.570154095701541;0.35036496350364965;0.5344687753446877
MotorImagery;0.53;0.59;0.5;0.44;0.5;0.5;0.36;0.5;0.61;0.54;;;;0.53;0.55;0.58;0.5;;0.59;0.48;0.0
NATOPS;0.8833333333333333;0.8611111111111112;0.8555555555555555;0.8444444444444444;0.8833333333333333;0.8833333333333333;0.8166666666666667;0.8444444444444444;0.8888888888888888;0.8944444444444445;0.9611111111111111;0.8722222222222222;0.9055555555555556;0.9777777777777777;0.8388888888888889;0.8888888888888888;0.8722222222222222;0.8111111111111111;0.8388888888888889;0.8;0.9611111111111111
PenDigits;0.9825614636935391;0.9082332761578045;0.967409948542024;0.9765580331618068;0.9771297884505432;0.9771297884505432;0.9725557461406518;0.9353916523727844;0.9339622641509434;0.9791309319611207;0.9877072612921669;0.9445397369925672;0.9668381932532876;0.9854202401372213;0.8319039451114922;0.9831332189822756;0.9405374499714122;0.8556317895940537;0.9353916523727844;0.8922241280731846;0.9857061177815895
PEMS-SF;0.8265895953757225;0.9826589595375722;1.0;1.0;0.7341040462427746;0.7109826589595376;0.7398843930635838;0.9075144508670521;0.9826589595375722;1.0;;;;0.7976878612716763;0.9942196531791907;0.8323699421965318;0.9710982658959537;;1.0;0.9826589595375722;0.0
PhonemeSpectra;0.27676707426185504;0.1947509692812407;0.2654339397554429;0.3077840739636147;0.15120787354607815;0.15120787354607815;0.10468237399343872;0.22397852669251417;0.3206084103787653;0.29048613182224875;;;;0.30002982403817474;0.26931106471816285;0.2755741127348643;0.29495973754846405;;0.24604831494184312;0.13719057560393677;0.16045332538025647
RacketSports;0.9013157894736842;0.881578947368421;0.881578947368421;0.9013157894736842;0.8421052631578947;0.8026315789473685;0.8486842105263158;0.881578947368421;0.8881578947368421;0.9078947368421053;0.881578947368421;0.8947368421052632;0.9276315789473685;0.9078947368421053;0.8092105263157895;0.8947368421052632;0.8881578947368421;0.875;0.8355263157894737;0.8881578947368421;0.8289473684210527
SelfRegulationSCP1;0.8464163822525598;0.8054607508532423;0.8600682593856656;0.8771331058020477;0.7849829351535836;0.7747440273037542;0.7747440273037542;0.8225255972696246;0.8532423208191127;0.8907849829351536;0.8395904436860068;0.7610921501706485;0.6962457337883959;0.7679180887372014;0.7235494880546075;0.8430034129692833;0.8395904436860068;0.9351535836177475;0.8122866894197952;0.8395904436860068;0.0
SelfRegulationSCP2;0.49444444444444446;0.4888888888888889;0.5;0.49444444444444446;0.5222222222222223;0.5388888888888889;0.4722222222222222;0.5166666666666667;0.46111111111111114;0.5;0.4722222222222222;0.4666666666666667;0.5277777777777778;0.4666666666666667;0.49444444444444446;0.5722222222222222;0.5333333333333333;0.48333333333333334;0.5;0.48333333333333334;0.4888888888888889
StandWalkJump;0.5333333333333333;0.3333333333333333;0.4;0.5333333333333333;0.3333333333333333;0.2;0.26666666666666666;0.3333333333333333;0.3333333333333333;0.4666666666666667;0.4;0.3333333333333333;0.26666666666666666;0.3333333333333333;0.26666666666666666;0.5333333333333333;0.4666666666666667;0.13333333333333333;0.3333333333333333;0.3333333333333333;0.0
UWaveGestureLibrary;0.928125;0.85625;0.925;0.909375;0.9;0.903125;0.865625;0.896875;0.890625;0.928125;0.878125;0.91875;0.93125;0.865625;0.684375;0.940625;0.85;0.9;0.925;0.775;0.0
58 changes: 58 additions & 0 deletions examples/benchmark/time_series_multi_clf_benchmark.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import pandas as pd
from sklearn.metrics import accuracy_score

from examples.benchmark.time_series_multi_reg_benchmark import evaluate_loop
from fedot_ind.core.optimizer.IndustrialEvoOptimizer import IndustrialEvoOptimizer
from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels
from tsml_eval._wip.results.results_by_classifier import *

available_operations = [
'eigen_basis',
'dimension_reduction',
'inception_model',
'logit',
'rf',
'xgboost',
'minirocket_extractor',
'normalization',
'omniscale_model',
'pca',
'mlp',
'quantile_extractor',
# 'resample',
'scaling',
'signal_extractor',
'topological_features'
]
experiment_setup = {'task': 'classification',
'metric': 'accuracy',
'timeout': 30,
'available_operations': available_operations,
'n_jobs': 2,
'max_pipeline_fit_time': 4,
'optimizer': IndustrialEvoOptimizer}

dataset_list = multivariate_equal_length
metric_dict = {}

if __name__ == "__main__":
OperationTypesRepository = IndustrialModels().setup_repository()
try:
results = pd.read_csv('./time_series_multi_reg_comparasion.csv', sep=';', index_col=0)
except Exception:
results = get_averaged_results_from_web(datasets=multivariate_equal_length, classifiers=valid_multi_classifiers)
results = pd.DataFrame(results)
results.columns = valid_multi_classifiers
results.index = multivariate_equal_length

results['Fedot_Ind'] = 0
for dataset in dataset_list:
prediction, target = evaluate_loop(dataset, experiment_setup)
try:
metric = accuracy_score(y_true=target, y_pred=prediction)
except Exception:
metric = accuracy_score(y_true=target, y_pred=np.argmax(prediction, axis=1))
metric_dict.update({dataset: metric})
results.loc[dataset, 'Fedot_Ind'] = metric
results.to_csv('./time_series_multi_clf_industrial_run.csv')
print(metric_dict)
Empty file added examples/benchmark/time_series_multi_forecast_benchmark.py
View file
Empty file.
132 changes: 132 additions & 0 deletions examples/benchmark/time_series_multi_reg_benchmark.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
import matplotlib
import pandas as pd
from fedot import Fedot

from examples.example_utils import init_input_data
from fedot_ind.core.optimizer.IndustrialEvoOptimizer import IndustrialEvoOptimizer
from fedot_ind.tools.loader import DataLoader
from fedot_ind.core.repository.initializer_industrial_models import IndustrialModels
from tsml_eval._wip.results.results_by_classifier import *
from statsmodels.tools.eval_measures import rmse


def evaluate_loop(dataset, experiment_setup: dict = None):
matplotlib.use('TkAgg')
train_data, test_data = DataLoader(dataset_name=dataset).load_data()
input_data = init_input_data(train_data[0], train_data[1], task=experiment_setup['task'])
val_data = init_input_data(test_data[0], test_data[1], task=experiment_setup['task'])

model = Fedot(problem=experiment_setup['task'],
logging_level=20,
n_jobs=experiment_setup['n_jobs'],
metric=experiment_setup['metric'],
pop_size=20,
num_of_generations=20,
optimizer=experiment_setup['optimizer'],
available_operations=experiment_setup['available_operations'],
max_pipeline_fit_time=experiment_setup['max_pipeline_fit_time'],
timeout=experiment_setup['timeout'],
with_tuning=False
)
model.fit(input_data)
prediction = model.predict(val_data)
try:
model.history.save(f"{dataset}_optimisation_history.json")
model.current_pipeline.show(save_path=f'./{dataset}_best_model.png')
model.history.show.operations_animated_bar(save_path=f'./{dataset}_history_animated_bars.gif',
show_fitness=True, dpi=100)
except Exception:
print('No_visualisation')
return prediction, val_data.target


# Regression equal length no missing problems
monash_regression = [
"AppliancesEnergy",
"AustraliaRainfall",
"BIDMC32HR",
"BIDMC32RR",
"BIDMC32SpO2",
"Covid3Month",
"FloodModeling1",
"FloodModeling2",
"FloodModeling3",
"IEEEPPG",
"LiveFuelMoistureContent",
"NewsHeadlineSentiment",
"NewsTitleSentiment",
"PPGDalia",
]

# 19 Regression problems with interpolated missing and truncated unequal
monash_regression_nm_eq = [
"AppliancesEnergy",
"AustraliaRainfall",
"BeijingPM10Quality-no-missing",
"BeijingPM25Quality-no-missing",
"BenzeneConcentration-no-missing",
"BIDMC32HR",
"BIDMC32RR",
"BIDMC32SpO2",
"Covid3Month",
"FloodModeling1",
"FloodModeling2",
"FloodModeling3",
"HouseholdPowerConsumption1-no-missing",
"HouseholdPowerConsumption2-no-missing",
"IEEEPPG",
"LiveFuelMoistureContent",
"NewsHeadlineSentiment",
"NewsTitleSentiment",
"PPGDalia-equal-length",
]

available_operations = ['rfr',
'ridge',
'gbr',
'sgdr',
'lgbmreg',
'linear',
'xgbreg',
'dtreg',
'treg',
'knnreg',
'scaling',
'normalization',
'pca',
'kernel_pca',
'isolation_forest_reg',
'eigen_basis',
'fourier_basis',
#'dimension_reduction',
'minirocket_extractor',
'quantile_extractor',
'signal_extractor',
'topological_features'
]

experiment_setup = {'task': 'regression',
'metric': 'rmse',
'timeout': 30,
'available_operations': available_operations,
'n_jobs': 2,
'max_pipeline_fit_time': 4,
'optimizer': IndustrialEvoOptimizer}

dataset_list = monash_regression
metric_dict = {}

if __name__ == "__main__":
OperationTypesRepository = IndustrialModels().setup_repository()
results = pd.read_csv('./time_series_multi_reg_comparasion.csv', sep=';', index_col=0)
results = results.dropna(axis=1, how='all')
results = results.dropna(axis=0, how='all')
results['Fedot_Ind'] = 0
for dataset in dataset_list:
prediction, target = evaluate_loop(dataset, experiment_setup)
metric = rmse(target, prediction)[0]
metric_dict.update({dataset: metric})
results.loc[dataset, 'Fedot_Ind'] = metric
results.to_csv('./time_series_multi_reg_industrial_run.csv')
print(metric_dict)
_ = 1
21 changes: 21 additions & 0 deletions examples/benchmark/time_series_multi_reg_comparasion.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
Dataset Name;FPCR;FPCR-Bspline;SVR;SVR Optimised;Random Forest;XGBoost;1-NN-ED;5-NN-ED;1-NN-DTWD;5-NN-DTWD;Rocket;FCN;ResNet;Inception;;;;;;
HouseholdPowerConsumption1;147,548998;147,5492;519,155544;152,391358;248,858964;231,089829;473,932736;432,594707;427,04311;297,221675;132,798779;162,244492;193,207281;153,716402;;;;;;
PPGDalia;20,674488;20,674486;19,005216;19,005216;17,530628;16,58273;21,876567;18,282277;26,024576;20,768389;14,050544;13,038805;11,382165;9,923701;;;;;;
AppliancesEnergy;5,405052;5,405052;3,4576312;3,454574;3,4551198;3,489024;5,231953;4,227438;6,036547;4,019873;2,2990312;2,865684;3,065047;4,43533;;;;;;
HouseholdPowerConsumption2;46,925185;46,929783;57,339837;55,98083;46,932139;44,3729326;71,479369;64,272956;58,802634;51,494969;32,607104;46,829256;39,080121;39,409826;;;;;;
IEEEPPG;31,381214;31,381212;36,301933;37,254146;32,10907;31,487901;33,208862;27,111213;37,140393;33,572786;36,5154892;34,325728;33,150985;23,903929;;;;;;
FloodModeling1;0,018853;0,018853;0,077897;0,046304;0,015891;0,0159712;0,01482;0,016193;0,011689;0,009801;0,002356;0,006709;0,008868;0,01743;;;;;;
BeijingPM25Quality;69,379217;69,369892;75,734162;71,437076;63,301428;59,495865;88,193545;74,156382;88,256082;72,717689;62,769655;59,726847;64,462746;62,227924;;;;;;
BenzeneConcentration;11,088396;11,094974;4,790901;4,790901;0,855559;0,6377256;6,535685;5,84498;4,983578;4,868465;3,360614;4,988295;4,0612608;1,584852;;;;;;
FloodModeling3;0,021458;0,021458;0,035032;0,035032;0,020429;0,0207038;0,019947;0,020765;0,01375;0,013337;0,004064;0,007873;0,01558;0,00821;;;;;;
BeijingPM10Quality;99,725946;99,732125;110,574226;110,574226;94,072344;93,138127;139,22979;115,669411;139,134908;115,502744;120,0577646;94,348729;95,489374;96,749997;;;;;;
FloodModeling2;0,019079;0,019079;0,075804;0,075804;0,014095;0,018199;0,018552;0,018586;0,016356;0,016238;0,005881;0,006719;0,013939;0,00729;;;;;;
AustraliaRainfall;8,436335;8,436336;8,650856;8,650856;8,389541;8,492986;30,254139;10,232841;12,001981;11,95073;8,124137333;8,425874;8,179173;8,841251;;;;;;
NewsHeadlineSentiment;0,142273;0,142272;0,142917;0,142917;0,147582;0,142486;0,202821;0,156636;0,197937;0,155839;0,142244;0,148065;0,150024;0,150014;;;;;;
NewsTitleSentiment;0,138126;0,138126;0,138881;0,138881;0,143103;0,138336;0,193318;0,15095;0,187257;0,150564;0,138059;0,138082;0,138295;0,158558;;;;;;
LiveFuelMoistureContent;37,683857;37,688074;43,021947;39,733527;32,1626252;32,441886;47,836798;38,535526;39,971707;35,185301;29,4097538;33,25722;30,3516564;28,796294;;;;;;
BIDMC32SpO2;4,953519;4,953517;4,818862;4,796855;4,570262;4,450805;5,530202;5,407875;5,215027;5,123964;5,221737;5,968337;5,987832;5,57612;;;;;;
BIDMC32HR;13,980558;13,980597;13,579905;13,39297;15,016468;13,963799;14,836506;14,756088;15,29101;15,127008;13,9443828;13,130665;10,74142;9,424679;;;;;;
BIDMC32RR;3,364777;3,364704;4,160427;3,17366;4,350314;4,367828;4,387345;4,134685;3,529111;3,432247;4,0929006;3,577775;3,921214;3,018405;;;;;;
Covid3Month;0,044912;0,044912;0,06584;0,06584;0,0424;0,044682;0,05306;0,041815;0,052735;0,042943;0,0438782;0,07434;0,095338;0,053769;;;;;;
;;;;;;;;;;;;;;;;;;;;
Empty file added examples/benchmark/time_series_uni_clf_benchmark.py
View file
Empty file.
Empty file added examples/benchmark/time_series_uni_forecast_benchmark.py
View file
Empty file.
Loading

0 comments on commit 054c0b4

Please sign in to comment.