Skip to content

Commit

Permalink
add results d=0 score manual and boxplots
Browse files Browse the repository at this point in the history
  • Loading branch information
@MathNog
MathNog committed Jul 9, 2024
1 parent c5a14fd commit 29b8894
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 32 deletions.
5 changes: 3 additions & 2 deletions boxplots.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ CSV.write("statistics_results_expressions_score_manual_d1.csv", results; delim='
df_freq_status = get_freq_status(data)
CSV.write("freq_status_expressions_score_manual_d1.csv", df_freq_status; delim=';', decimal=',')

println("Tempo total = ",(sum(data[:, "t_create"]) + sum(data[:, "t_optim"])) / 60, " seg")
println("Tempo total = ",(sum(data[:, "t create"]) + sum(data[:, "t optim"])) / 60, " seg")

# Rename columns to be Julia friendly
rename!(data, Dict(Symbol("t create") => :t_create))
Expand All @@ -67,4 +67,5 @@ box3 = @df data boxplot(:model, :log_rmse_train, legend=false, xlabel="Modelo",
box4 = @df data boxplot(:model, :log_rmse_test, legend=false, xlabel="Modelo", ylabel="RMSE", title="Boxplot RMSE de teste por modelo")
box5 = @df data boxplot(:model, :log_mase_test, legend=false, xlabel="Modelo", ylabel="MASE", title="Boxplot MASE de teste por modelo")

plot(box1, box2, box3, box4, box5, layout=(3,2), size=(1500,700), margin = 7Plots.mm)
plot(box1, box2, box3, box4, box5, layout=(3,2), size=(1500,700), margin = 7Plots.mm, suptitle = "Score Manual d=1")
savefig("boxplots_score_manual_d1.png")
Binary file added boxplots_score_MEB.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added boxplots_score_manual_d0.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added boxplots_score_manual_d1.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
8 changes: 8 additions & 0 deletions freq_status_expressions_score_manual_d0.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
model;status;count;freq
model 1;ITERATION_LIMIT;17;0,17
model 1;LOCALLY_SOLVED;71;0,71
model 1;ALMOST_LOCALLY_SOLVED;12;0,12
model 2;LOCALLY_SOLVED;99;0,99
model 2;NORM_LIMIT;1;0,01
model 3;LOCALLY_SOLVED;99;0,99
model 3;NORM_LIMIT;1;0,01
78 changes: 78 additions & 0 deletions results_variables_expressions_MEB_score_manual_d0.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,3 +221,81 @@ serie,T,model,t create,t optim,rmse train,rmse test,mase test,status
74,170,model 1,0.014121,0.7871735,2499.7123032745253,19334.82538912702,9.455639484685026,LOCALLY_SOLVED
74,170,model 2,0.0045628,0.1931604,3370.6681494934974,71889.62768450989,34.42210324198075,LOCALLY_SOLVED
74,170,model 3,0.0046391,0.4930121,2499.9441808116667,19280.874808529792,9.431853422741485,LOCALLY_SOLVED
75,170,model 1,0.0134547,79.2004613,644470.799213985,5.098198222685922e7,64.22173259004238,ITERATION_LIMIT
75,170,model 2,0.0044523,0.0260546,661241.243145347,2.66455552999646e6,3.7510076831490653,LOCALLY_SOLVED
75,170,model 3,0.0045862,0.0337213,645080.9010513237,3.002336912291349e6,4.203625789162262,LOCALLY_SOLVED
76,170,model 1,0.0138403,0.9777318,2609.536030603713,32445.739311523343,9.138021903877613,LOCALLY_SOLVED
76,170,model 2,0.0050375,1.9347579,2682.878927234288,39229.89067934093,11.070593416256205,LOCALLY_SOLVED
76,170,model 3,0.0043896,0.5758593,2609.3309697327777,32465.56778427494,9.145124925475272,LOCALLY_SOLVED
77,170,model 1,0.0144024,0.7594656,272.4333190694481,16977.4776144125,8.152206608856345,LOCALLY_SOLVED
77,170,model 2,0.0043043,0.8209858,273.33447016360077,17281.77892710933,8.300549946501423,LOCALLY_SOLVED
77,170,model 3,0.0044327,0.3184717,271.9334370996856,16988.484862874822,8.159639227906638,LOCALLY_SOLVED
78,170,model 1,0.0136055,0.3653964,577.6351232577509,2157.795622806721,1.2625898510733347,LOCALLY_SOLVED
78,170,model 2,0.0052634,0.8269645,579.0892118357681,2561.9348235158186,1.487335301121484,LOCALLY_SOLVED
78,170,model 3,0.0068339,0.402072,578.0130070484291,2213.7361230161246,1.29618650666323,LOCALLY_SOLVED
79,170,model 1,0.0129186,1.2523475,3.168576922287069e-5,8.406765304447589,0.009727873333052629,LOCALLY_SOLVED
79,170,model 2,0.0053178,0.6011807,0.00015371414340986506,26.579563099870782,0.03136427360956223,LOCALLY_SOLVED
79,170,model 3,0.0059569,0.8861268,0.00015710222671415345,46.67727521735619,0.05316248092331488,LOCALLY_SOLVED
80,170,model 1,0.0170495,0.6013343,2554.3793374529687,205628.73866277927,55.36783368801589,LOCALLY_SOLVED
80,170,model 2,0.0056146,0.5065897,2572.7234135649364,209942.34286397012,56.542622526179926,LOCALLY_SOLVED
80,170,model 3,0.0257713,0.1593784,2544.562874901234,205414.71669627345,55.29776879624208,LOCALLY_SOLVED
81,170,model 1,0.0143732,0.9867096,9.096174790879628e-5,107.80154597652954,0.28463657597161246,LOCALLY_SOLVED
81,170,model 2,0.0048266,0.3134621,0.00010637604448033036,175.17328023215853,0.47688044029810767,LOCALLY_SOLVED
81,170,model 3,0.0049682,0.784724,0.00010205260674716895,107.44862612062032,0.2910198138175132,LOCALLY_SOLVED
82,170,model 1,0.0323436,1.1822311,0.00011398036853242935,125.59049219190733,0.04814423073060329,LOCALLY_SOLVED
82,170,model 2,0.004667,1.8297485,0.00012170245057910213,118.24993859225383,0.04573577339102412,LOCALLY_SOLVED
82,170,model 3,0.0048116,0.0505767,174.6812736597607,2312.771210484296,0.9836783190201449,LOCALLY_SOLVED
83,170,model 1,0.0179533,0.4299301,816.299505395996,16773.87061248336,14.215578184159286,LOCALLY_SOLVED
83,170,model 2,0.0047192,0.3324792,820.7600979877851,17692.459846314992,14.985562664832772,LOCALLY_SOLVED
83,170,model 3,0.0042319,1.1528254,816.2384623539131,16618.28410194924,14.083327129598866,LOCALLY_SOLVED
84,170,model 1,0.0123449,0.5623278,2554.3793374529687,205628.73866277927,55.36783368801589,LOCALLY_SOLVED
84,170,model 2,0.004437,0.5041899,2572.7234135649364,209942.34286397012,56.542622526179926,LOCALLY_SOLVED
84,170,model 3,0.0046708,0.1794781,2544.562874901234,205414.71669627345,55.29776879624208,LOCALLY_SOLVED
85,170,model 1,0.0142273,1.2840549,2241.34330086283,215938.65758856267,95.31926345482722,LOCALLY_SOLVED
85,170,model 2,0.0238452,0.501279,2245.788255912496,214600.68039905347,94.72802228589204,LOCALLY_SOLVED
85,170,model 3,0.0053187,0.4141567,2242.4858630960634,215931.506020039,95.31375346705964,LOCALLY_SOLVED
86,170,model 1,0.0333923,1.0389155,1904.2719521345944,229524.64380254224,195.64716105530604,ALMOST_LOCALLY_SOLVED
86,170,model 2,0.0043049,4.1931937,1904.9216958057655,228244.65068971273,194.5275139538946,LOCALLY_SOLVED
86,170,model 3,0.0047008,1.4967917,1904.5874360921084,229617.36986479524,195.72238178106863,LOCALLY_SOLVED
87,170,model 1,0.0128633,2.9637461,590168.1358475862,2.5266169773026328e6,6.094242008245338,ALMOST_LOCALLY_SOLVED
87,170,model 2,0.0042603,0.0522826,592686.2133606094,969773.0221434354,2.403047626918803,LOCALLY_SOLVED
87,170,model 3,0.0044839,0.0726814,811200.9366163167,1.346967092818444e6,3.564897875547578,LOCALLY_SOLVED
88,170,model 1,0.0121346,17.2978583,1.012684102881733e7,1.01696079871078e7,0.4627204033145247,ITERATION_LIMIT
88,170,model 2,0.0045686,0.0287574,1.0615108175163386e7,7.847797133289884e6,0.27719999450686517,LOCALLY_SOLVED
88,170,model 3,0.0045959,0.0454728,1.0049496998343945e7,4.645819201438846e6,0.19389246480440087,LOCALLY_SOLVED
89,170,model 1,0.0343647,0.5340144,1027.4351010009439,49222.511481274436,36.102830798067195,LOCALLY_SOLVED
89,170,model 2,0.005637,0.4417373,1027.3158871486085,48982.344768455434,35.91621822221207,LOCALLY_SOLVED
89,170,model 3,0.0048572,0.3601371,1027.8710085708026,49451.119855633755,36.256017266068014,LOCALLY_SOLVED
90,170,model 1,0.0123714,0.7506173,272.4333190694481,16977.4776144125,8.152206608856345,LOCALLY_SOLVED
90,170,model 2,0.0040777,0.7853896,273.33447016360077,17281.77892710933,8.300549946501423,LOCALLY_SOLVED
90,170,model 3,0.0043793,0.3129078,271.9334370996856,16988.484862874822,8.159639227906638,LOCALLY_SOLVED
91,170,model 1,0.014792,1.2239506,2241.34330086283,215938.65758856267,95.31926345482722,LOCALLY_SOLVED
91,170,model 2,0.0051139,0.5021408,2245.788255912496,214600.68039905347,94.72802228589204,LOCALLY_SOLVED
91,170,model 3,0.0074031,0.4070787,2242.4858630960634,215931.506020039,95.31375346705964,LOCALLY_SOLVED
92,170,model 1,0.0134446,0.6462712,889.4927114737117,74064.66190951449,79.33627703242811,LOCALLY_SOLVED
92,170,model 2,0.0046633,0.51687,889.6734599923165,73524.5705598398,78.78935245189783,LOCALLY_SOLVED
92,170,model 3,0.0056224,0.7205339,885.7576250822397,71845.80724435743,76.9949788623738,LOCALLY_SOLVED
93,170,model 1,0.0137827,1.0460169,1904.2719521345944,229524.64380254224,195.64716105530604,ALMOST_LOCALLY_SOLVED
93,170,model 2,0.0046943,4.1230363,1904.9216958057655,228244.65068971273,194.5275139538946,LOCALLY_SOLVED
93,170,model 3,0.0054675,1.5078258,1904.5874360921084,229617.36986479524,195.72238178106863,LOCALLY_SOLVED
94,170,model 1,0.033798,0.9894741,2609.536030603713,32445.739311523343,9.138021903877613,LOCALLY_SOLVED
94,170,model 2,0.0045908,1.9142476,2682.878927234288,39229.89067934093,11.070593416256205,LOCALLY_SOLVED
94,170,model 3,0.0045946,0.5377373,2609.3309697327777,32465.56778427494,9.145124925475272,LOCALLY_SOLVED
95,170,model 1,0.0129081,0.4518503,57.8409370613198,1078.9274217025359,4.267402836538898,LOCALLY_SOLVED
95,170,model 2,0.0044469,0.216765,57.61393744195771,1028.0892291289863,4.059861572244256,LOCALLY_SOLVED
95,170,model 3,0.0044308,0.2926116,57.58445632268189,1002.3552939786113,3.959214233160198,LOCALLY_SOLVED
96,170,model 1,0.0125973,0.3302966,492.15347123306566,35683.683154626124,16.463937862626743,LOCALLY_SOLVED
96,170,model 2,0.0041117,0.2474654,495.05004014481324,34624.47012026341,15.976193574946581,LOCALLY_SOLVED
96,170,model 3,0.0045486,0.3616671,492.16220472649127,35680.82492526758,16.462557944622674,LOCALLY_SOLVED
97,170,model 1,0.0339348,0.9920092,9.096174790879628e-5,107.80154597652954,0.28463657597161246,LOCALLY_SOLVED
97,170,model 2,0.0046462,0.3126673,0.00010637604448033036,175.17328023215853,0.47688044029810767,LOCALLY_SOLVED
97,170,model 3,0.0047745,0.7455558,0.00010205260674716895,107.44862612062032,0.2910198138175132,LOCALLY_SOLVED
98,170,model 1,0.0124351,0.3186551,492.15347123306566,35683.683154626124,16.463937862626743,LOCALLY_SOLVED
98,170,model 2,0.0239651,0.2762434,495.05004014481324,34624.47012026341,15.976193574946581,LOCALLY_SOLVED
98,170,model 3,0.0046951,0.3925645,492.16220472649127,35680.82492526758,16.462557944622674,LOCALLY_SOLVED
99,170,model 1,0.0147676,0.4486852,400.045613718239,13112.553232713211,8.18169127477447,LOCALLY_SOLVED
99,170,model 2,0.0041602,0.0678051,430.11392306302906,23988.971690203423,14.94426174850968,LOCALLY_SOLVED
99,170,model 3,0.0044887,0.2128308,399.817603303143,12020.972209102652,7.506868369677366,LOCALLY_SOLVED
100,170,model 1,0.0149717,3.5581656,397629.3781497035,3.8841065007188424e7,72.23061453989442,ALMOST_LOCALLY_SOLVED
100,170,model 2,0.0050756,0.2934176,455003.0096189513,1.6262500493570393e7,30.394793959741346,LOCALLY_SOLVED
100,170,model 3,0.004303,0.033871,398333.09777988744,1.4629077477203219e6,3.1574765640564806,LOCALLY_SOLVED
45 changes: 15 additions & 30 deletions statistics_results_expressions_MEB.csv
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,16 @@
model;metric;mean;std;median
model 1;t_create;0,017652966;0,009151609551674891;0,01395575
model 1;t_optim;22,785713202999997;19,312309502324783;19,97279255
model 1;rmse_train;1,2197984610107755e6;6,729781187971417e6;49,92174098350877
model 1;rmse_test;1,6563450902643e6;1,063993046968673e7;1011,6431396780729
model 1;mase_test;0,8314255325456549;1,5757315810490742;0,3172115577663968
model 1;log_t_create;-4,123055914571561;0,37272845324005277;-4,271863742740658
model 1;log_t_optim;2,540832941280826;1,3172629267096985;2,9943709374693017
model 1;log_rmse_train;4,7411153982583505;4,506715706762381;3,910456598923966
model 1;log_rmse_test;7,549781968420996;3,72776061106393;6,919331158882656
model 1;log_mase_test;-1,2433903391995704;1,7699096979470434;-1,148186353042889
model 2;t_create;0,005705923000000001;0,003115278497946642;0,00516585
model 2;t_optim;5,5918674500000005;26,81378189630079;0,7365753500000001
model 2;rmse_train;1,1339766623560223e6;1,0637451201082911e7;18,921146339803236
model 2;rmse_test;2,03507247696529e6;2,3400360142328127e-10;2,0350724769652903e6
model 2;mase_test;1,519833928417244;3,7338004832292047;0,26759399598378014
model 2;log_t_create;-5,217540063081621;0,2489113704886878;-5,265686688758137
model 2;log_t_optim;-0,22915312545740327;1,6074225106566766;-0,3057440942985159
model 2;log_rmse_train;4,466351744041143;4,421368412510809;2,9401095128382844
model 2;log_rmse_test;14,526041991440984;0,0;14,526041991440984
model 2;log_mase_test;-1,1061516362991803;1,9218632127309443;-1,3182843875127255
model 3;t_create;0,005579878000000002;0,002668488939402435;0,0051982999999999994
model 3;t_optim;8,00595565;32,721003454603725;0,7789082
model 3;rmse_train;75222,18083464606;339072,31975110783;20,71693279043651
model 3;rmse_test;2,5639886444076556e6;0,0;2,5639886444076556e6
model 3;mase_test;4,023014308710007;13,25539389886138;0,44049396452586326
model 3;log_t_create;-5,226092880105824;0,2151971606189759;-5,259423711521887
model 3;log_t_optim;-0,2275591546930696;1,7567070220641234;-0,2498958410831587
model 3;log_rmse_train;4,430922588205975;4,587727490626741;3,0294840115276322
model 3;log_rmse_test;14,757074668154724;1,785305796991587e-15;14,757074668154722
model 3;log_mase_test;-0,9268738970005722;2,1687649918212517;-0,8198585351166506
model 1;t create;0,017652966;0,009151609551674891;0,01395575
model 1;t optim;22,785713202999997;19,312309502324783;19,97279255
model 1;rmse train;1,2197984610107755e6;6,729781187971417e6;49,92174098350877
model 1;rmse test;1,6563450902643e6;1,063993046968673e7;1011,6431396780729
model 1;mase test;0,8314255325456549;1,5757315810490742;0,3172115577663968
model 2;t create;0,005705923000000001;0,003115278497946642;0,00516585
model 2;t optim;5,5918674500000005;26,81378189630079;0,7365753500000001
model 2;rmse train;1,1339766623560223e6;1,0637451201082911e7;18,921146339803236
model 2;rmse test;2,03507247696529e6;2,3400360142328127e-10;2,0350724769652903e6
model 2;mase test;1,519833928417244;3,7338004832292047;0,26759399598378014
model 3;t create;0,005579878000000002;0,002668488939402435;0,0051982999999999994
model 3;t optim;8,00595565;32,721003454603725;0,7789082
model 3;rmse train;75222,18083464606;339072,31975110783;20,71693279043651
model 3;rmse test;2,5639886444076556e6;0,0;2,5639886444076556e6
model 3;mase test;4,023014308710007;13,25539389886138;0,44049396452586326
16 changes: 16 additions & 0 deletions statistics_results_expressions_score_manual_d0.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
model;metric;mean;std;median
model 1;t create;0,019091550000000002;0,012312040901520232;0,014116199999999999
model 1;t optim;11,417610491;26,905511276353693;0,96323585
model 1;rmse train;3,6166269690350695e6;2,7108137415244907e7;889,4927114737117
model 1;rmse test;1,1938164938324152e7;4,343009995941714e7;34474,8238060782
model 1;mase test;33,376286217284644;47,07491540586178;8,358160697413215
model 2;t create;0,012077398000000001;0,06721749197593695;0,00466245
model 2;t optim;0,523843026;0,8942545258559712;0,28866435
model 2;rmse train;2,021834114063781e6;1,1453799905230418e7;889,6734599923165
model 2;rmse test;7,998898288870259e6;3,632148668535795e7;39229,89067934093
model 2;mase test;29,916341641191888;47,63245793935258;6,138682602963707
model 3;t create;0,008853728;0,0323417715158282;0,0049305
model 3;t optim;0,479888577;0,8396992803914033;0,35397365000000003
model 3;rmse train;932183,3906138331;4,376235974652707e6;850,9980437180764
model 3;rmse test;6,916395296111653e6;3,394672979915958e7;35680,82492526758
model 3;mase test;28,805205407246728;48,14108143089217;5,931896644899783
16 changes: 16 additions & 0 deletions statistics_results_expressions_score_manual_d1.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
model;metric;mean;std;median
model 1;t create;0,019126108000000003;0,012894298462543339;0,013869
model 1;t optim;13,023301487000001;17,61567577509764;3,3401729
model 1;rmse train;1,3956413209135488e6;7,3654206362113785e6;31,987358380231704
model 1;rmse test;1,860484112900937e6;1,112588469762843e7;775,759431392069
model 1;mase test;1,1950864125847422;2,3000256392803395;0,27886496263974103
model 2;t create;0,005733680000000001;0,004821080385979973;0,0048909999999999995
model 2;t optim;3,4993549180000003;12,970775708581353;0,46174455000000003
model 2;rmse train;1,137513016620315e6;1,063710150570363e7;19,44383967778378
model 2;rmse test;2,0511001751000124e6;1,7432187580775533e7;1117,1116368863468
model 2;mase test;1,4741778760964168;4,092868672530054;0,30857957524149193
model 3;t create;0,005435776999999999;0,0030652395278677703;0,00507725
model 3;t optim;8,84116094;30,39674525020637;0,5324408
model 3;rmse train;77594,42933786417;339399,69673344534;19,251133641707725
model 3;rmse test;2,0615901167371592e6;1,6778471778635558e7;1611,5944448021778
model 3;mase test;4,975952321270998;19,22936808906624;0,3145265718716847

0 comments on commit 29b8894

Please sign in to comment.