Skip to content

Commit

Permalink
Merge pull request #35 from datagram-db/benchmarking
Browse files Browse the repository at this point in the history 
Added atomization_type to logger. Fixed atomization_file bug. ALl new…
  • Loading branch information
@SamuelAppleby
SamuelAppleby authored Nov 8, 2022
2 parents fb3c877 + c04e9ef commit 53cd566
Show file tree
Hide file tree
Showing 33 changed files with 6,063 additions and 3,155 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ add_executable(tests_generator tests/tests_generator.cpp) #${BACKWARD_ENABLE})
target_link_libraries(tests_generator knobab Threads::Threads
yaucl_stringutils yaucl_numeric yaucl_hashing yaucl_data yaucl_bpm yaucl_graph knobab_flloat yaucl_structures)

add_executable(benchmarking benchmarking/benchmarking.cpp benchmarking/benchmarking.cpp) #${BACKWARD_ENABLE})
add_executable(benchmarking benchmarking/benchmarking.cpp) #${BACKWARD_ENABLE})
target_link_libraries(benchmarking knobab Threads::Threads
yaucl_stringutils yaucl_numeric yaucl_hashing yaucl_data yaucl_bpm yaucl_graph knobab_flloat yaucl_structures)

Expand Down
37 changes: 18 additions & 19 deletions benchmarking/benchmarking.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
#include "knobab/operators/fast_ltlf_operators.h"
#include <fstream>

const std::string file_path = "data/testing/results/benchmarking/custom_operators.csv";
const std::string file_path = "data/results/custom_operators.csv";

Environment setup(const std::filesystem::path f){
Environment env;
Expand Down Expand Up @@ -119,19 +119,18 @@ void test_aAndFutureb(const Environment& environment){
auto t1 = std::chrono::high_resolution_clock::now();
auto t2 = std::chrono::high_resolution_clock::now();

// A&Ft_new(B)
t1 = std::chrono::high_resolution_clock::now();
aAndFutureB_timed_variant_2(a, b, r_aAndfb, nullptr, environment.db.act_table_by_act_id.trace_length);
t2 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> r_aAndfb_time = t2 - t1;
write_to_file(environment, "And_Future", "fast", r_aAndfb_time.count());

// A&Ft_old(B)
t1 = std::chrono::high_resolution_clock::now();
aAndFutureB_timed_variant_1(a, b, r_aAndfb, nullptr, environment.db.act_table_by_act_id.trace_length);
t2 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> r_aAndfb_old_time = t2 - t1;
write_to_file(environment, "And_Future", "fast_old", r_aAndfb_old_time.count());
write_to_file(environment, "And_Future", "fast_variant_1", r_aAndfb_old_time.count());

t1 = std::chrono::high_resolution_clock::now();
aAndFutureB_timed_variant_2(a, b, r_aAndfb, nullptr, environment.db.act_table_by_act_id.trace_length);
t2 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> r_aAndfb_time = t2 - t1;
write_to_file(environment, "And_Future", "fast_variant_2", r_aAndfb_time.count());

// A AND Ft(B)
t1 = std::chrono::high_resolution_clock::now();
Expand All @@ -151,19 +150,19 @@ void test_aAndGloballyb(const Environment& environment){
auto t1 = std::chrono::high_resolution_clock::now();
auto t2 = std::chrono::high_resolution_clock::now();

// A&G_new(B)
t1 = std::chrono::high_resolution_clock::now();
aAndGloballyB_timed_variant_2(a, b, r_aAndgb, nullptr, environment.db.act_table_by_act_id.trace_length);
t2 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> r_a_and_globally_b_ut_time = t2 - t1;
write_to_file(environment, "And_Globally", "fast", r_a_and_globally_b_ut_time.count());

// A&G_old(B)
t1 = std::chrono::high_resolution_clock::now();
aAndGloballyB_timed_variant_1(a, b, r_aAndgb, nullptr, environment.db.act_table_by_act_id.trace_length);
t2 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> r_a_and_globally_b_ut_time2 = t2 - t1;
write_to_file(environment, "And_Globally", "fast_old", r_a_and_globally_b_ut_time2.count());
write_to_file(environment, "And_Globally", "fast_variant_1", r_a_and_globally_b_ut_time2.count());

// A&G_new(B)
t1 = std::chrono::high_resolution_clock::now();
aAndGloballyB_timed_variant_2(a, b, r_aAndgb, nullptr, environment.db.act_table_by_act_id.trace_length);
t2 = std::chrono::high_resolution_clock::now();
std::chrono::duration<double, std::milli> r_a_and_globally_b_ut_time = t2 - t1;
write_to_file(environment, "And_Globally", "fast_variant_2", r_a_and_globally_b_ut_time.count());

// A AND Gt(B)
t1 = std::chrono::high_resolution_clock::now();
Expand Down Expand Up @@ -227,7 +226,7 @@ void test_until(const Environment& environment){
int main(){
const uint16_t iters = 5;

for (const auto& dirEntry : std::filesystem::recursive_directory_iterator("data/benchmarking/incomplete")){
for (const auto& dirEntry : std::filesystem::recursive_directory_iterator("data/synthetic")){
std::cout << dirEntry << std::endl;
std::filesystem::path curr = (dirEntry);
Environment environment = setup(curr);
Expand All @@ -237,7 +236,7 @@ int main(){
test_or(environment);
test_aAndFutureb(environment);
test_aAndGloballyb(environment);
test_aAndNextGloballyb(environment);
// test_aAndNextGloballyb(environment);
test_until(environment);
test_choice(environment);
}
Expand Down
83 changes: 83 additions & 0 deletions data/benchmarking_scripts/burattin.R
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
library(ggplot2)
library(reshape2)
library(reshape)
library(dplyr)
library(tidyr)
library(latex2exp)

setwd(dirname(rstudioapi::getActiveDocumentContext()$path))

knobabdf <- read.csv(file = '/home/sam/Documents/Repositories/CodeBases/knobab/data/results/burattin.csv')

knobabdf$execution_time <- knobabdf$model_data_decomposition_time + knobabdf$model_atomization_time + knobabdf$model_declare_to_ltlf + knobabdf$model_ltlf_query_time

knobabdf = knobabdf[,c('model_filename', 'atomization_conf', 'execution_time', 'model_type')]

knobabdf <- aggregate(list(execution_time = knobabdf$execution_time),
by=list(atomization_conf = knobabdf$atomization_conf,model_filename=knobabdf$model_filename,model_type=knobabdf$model_type),data=knobabdf,FUN=mean)

knobabdf[knobabdf == "scripts/intersection_pipeline.yaml"] <- "KnoBAB + CQ"
knobabdf[knobabdf == "scripts/support_pipeline.yaml"] <- "KnoBAB + Support"

burattindf <- read.csv(file = '/home/sam/Documents/Repositories/CodeBases/knobab/data/results/declare_analyzer.csv')

burattindf <- aggregate(list(execution_time = burattindf$execution_time),
by=list(model_filename=burattindf$model_filename,atomization_conf=burattindf$atomization_conf,model_type=burattindf$model_type),data=burattindf,FUN=mean)

combineddf <- rbind(knobabdf, burattindf)

scenario1df <- combineddf[combineddf$model_type == "SCENARIO_1", ]
scenario2df <- combineddf[combineddf$model_type == "SCENARIO_2", ]

scenario1df$atomization_conf <- factor(scenario1df$atomization_conf, levels=c("KnoBAB + CQ", "KnoBAB + Support", "Declare Analyzer"))
scenario2df$atomization_conf <- factor(scenario2df$atomization_conf, levels=c("KnoBAB + CQ", "KnoBAB + Support", "Declare Analyzer"))

ggplot(scenario1df, aes(x=reorder(model_filename, execution_time), y=execution_time, fill=atomization_conf)) +
geom_bar(position = "dodge", stat = "identity") +
labs(x = "Model", y = "Time (ms)") +
theme(legend.position="bottom") +
scale_x_discrete(labels=c("data/bpic_2012/models/scenario_1/5.powerdecl" = TeX(r'($M_1$)'),
"data/bpic_2012/models/scenario_1/10.powerdecl" = TeX(r'($M_2$)'),
"data/bpic_2012/models/scenario_1/15.powerdecl" = TeX(r'($M_3$)'),
"data/bpic_2012/models/scenario_1/20.powerdecl" = TeX(r'($M_4$)'))) +
scale_y_log10(
breaks = scales::trans_breaks("log10", function(x) 10^x),
labels = scales::trans_format("log10", scales::math_format(10^.x))) +
guides(fill=guide_legend(title="Atomization Type"))

yticks <- c(1, 10, 100, 10000)

# scenario2df$mean_t <- trans(scenario2df$execution_time)

scenario2df$model_filename <- factor(scenario2df$model_filename, levels=c("data/bpic_2012/models/scenario_2/q1.powerdecl",
"data/bpic_2012/models/scenario_2/q2.powerdecl",
"data/bpic_2012/models/scenario_2/q3.powerdecl",
"data/bpic_2012/models/scenario_2/q4.powerdecl",
"data/bpic_2012/models/scenario_2/q5.powerdecl",
"data/bpic_2012/models/scenario_2/q1^q2.powerdecl",
"data/bpic_2012/models/scenario_2/q1^q2^q4.powerdecl",
"data/bpic_2012/models/scenario_2/q1^q3^q4.powerdecl",
"data/bpic_2012/models/scenario_2/q1^q2^q5.powerdecl",
"data/bpic_2012/models/scenario_2/q1^q3^q5.powerdecl",
"data/bpic_2012/models/scenario_2/q1^q2^q3^q4^q5.powerdecl"))

ggplot(scenario2df, aes(x=model_filename, y=execution_time, fill=atomization_conf)) +
geom_col(position = "dodge") +
guides(fill=guide_legend(title="")) +
labs(x = TeX(r'($M$)'), y = "Execution Time (ms)") +
theme(legend.position="bottom") +
scale_x_discrete(labels=c("data/bpic_2012/models/scenario_2/q1.powerdecl" = TeX(r'($M_1$)'),
"data/bpic_2012/models/scenario_2/q2.powerdecl" = TeX(r'($M_2$)'),
"data/bpic_2012/models/scenario_2/q3.powerdecl" = TeX(r'($M_3$)'),
"data/bpic_2012/models/scenario_2/q4.powerdecl" = TeX(r'($M_4$)'),
"data/bpic_2012/models/scenario_2/q5.powerdecl" = TeX(r'($M_5$)'),
"data/bpic_2012/models/scenario_2/q1^q2.powerdecl" = TeX(r'($M_6$)'),
"data/bpic_2012/models/scenario_2/q1^q2^q4.powerdecl" = TeX(r'($M_7$)'),
"data/bpic_2012/models/scenario_2/q1^q3^q4.powerdecl" = TeX(r'($M_8$)'),
"data/bpic_2012/models/scenario_2/q1^q2^q5.powerdecl" = TeX(r'($M_9$)'),
"data/bpic_2012/models/scenario_2/q1^q3^q5.powerdecl" = TeX(r'($M_{10}$)'),
"data/bpic_2012/models/scenario_2/q1^q2^q3^q4^q5.powerdecl" = TeX(r'($M_{11}$)'))) +
scale_y_log10(
breaks = scales::trans_breaks("log10", function(x) 10^x),
labels = scales::trans_format("log10", scales::math_format(10^.x))) +
guides(fill=guide_legend(title="Atomization Type"))
2 changes: 1 addition & 1 deletion ...enchmarking_scripts/burattin_benchmark.sh → data/benchmarking_scripts/burattin.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
cd ..
cd ..

declare -a models=("data/bpic_2012/models/scenario_1/5.powerdecl" "data/bpic_2012/models/scenario_1/10.powerdecl" "data/bpic_2012/models/scenario_1/15.powerdecl" "data/bpic_2012/models/scenario_1/20.powerdecl" "data/bpic_2012/models/scenario_2/q1.powerdecl" "data/bpic_2012/models/scenario_2/scenario_2/q2.powerdecl" "data/bpic_2012/models/scenario_2/q3.powerdecl" "data/bpic_2012/models/scenario_2/q4.powerdecl" "data/bpic_2012/models/scenario_2/q5.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2^q3^q4^q5.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2^q4.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2^q5.powerdecl" "data/bpic_2012/models/scenario_2/q1+q3+q4.powerdecl" "data/bpic_2012/models/scenario_2/q1^q3^q5.powerdecl")
declare -a models=("data/bpic_2012/models/scenario_1/5.powerdecl" "data/bpic_2012/models/scenario_1/10.powerdecl" "data/bpic_2012/models/scenario_1/15.powerdecl" "data/bpic_2012/models/scenario_1/20.powerdecl" "data/bpic_2012/models/scenario_2/q1.powerdecl" "data/bpic_2012/models/scenario_2/q2.powerdecl" "data/bpic_2012/models/scenario_2/q3.powerdecl" "data/bpic_2012/models/scenario_2/q4.powerdecl" "data/bpic_2012/models/scenario_2/q5.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2^q3^q4^q5.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2^q4.powerdecl" "data/bpic_2012/models/scenario_2/q1^q2^q5.powerdecl" "data/bpic_2012/models/scenario_2/q1^q3^q4.powerdecl" "data/bpic_2012/models/scenario_2/q1^q3^q5.powerdecl")

declare -a pipelines=("scripts/intersection_pipeline.yaml" "scripts/support_pipeline.yaml")

Expand Down
56 changes: 0 additions & 56 deletions data/benchmarking_scripts/burattin_benchmarker.R
View file

This file was deleted.

Binary file removed data/benchmarking_scripts/choice.png
View file
Binary file not shown.
29 changes: 16 additions & 13 deletions ...nchmarking_scripts/operator_benchmarker.R → data/benchmarking_scripts/custom_operators.R
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ library(latex2exp)

setwd(dirname(rstudioapi::getActiveDocumentContext()$path))

df <- read.csv(file = '/home/giacomo/projects/knobab/data/testing/results/benchmarking/custom_operators.csv')
df <- read.csv(file = '/home/sam/Documents/Repositories/CodeBases/knobab/data/results/custom_operators.csv')

df <- aggregate(list(Time = df$Time),
by=list(Test = df$Test,Type=df$Type,Log_Size=df$Log_Size,Max_Trace_Length=df$Max_Trace_Length,Alphabet=df$Alphabet),data=df,FUN=mean)

print_chart <- function(df, labels, type.labs){
print_chart <- function(df, labels, type.labs, scale){
# test.labs <- c(TeX(r'(x)'), TeX(r'(x)'), TeX(r'(x)'))
test = c("hello", "there", "i", "am", "sam")
test1 = lapply(test, TeX)
Expand All @@ -29,19 +29,20 @@ print_chart <- function(df, labels, type.labs){
ggplot(df, aes(x=Log_Size, y=Time, shape=Type, color=Max_Trace_Length)) +
geom_line() +
geom_point(size=3) +
labs(x = "Log Size", y = "Time (ms)") +
labs(x = TeX(r'($ |{L}| $)'), y = "Execution Time (ms)") +
facet_wrap( ~ Test, nrow=2, ncol=2, labeller = labeller(Test = labels)) +
theme(legend.position="bottom") +
scale_color_discrete(name = TeX(r'($Max Trace Length$)')) +
scale_shape_discrete(labels=type.labs, name = TeX(r'($Algorithm$)')) +
scale_y_continuous(trans='log10') +
scale_color_discrete(name = TeX(r'($\epsilon$)')) +
scale_shape_discrete(labels=type.labs, name = "Algorithm") +
scale_y_continuous(trans=scale) +
scale_x_log10(
breaks = scales::trans_breaks("log10", function(x) 10^x),
labels = scales::trans_format("log10", scales::math_format(10^.x))) +
guides(colour = guide_legend(order = 1), shape = guide_legend(order = 2))
}

df$Max_Trace_Length <- as.character(df$Max_Trace_Length)
df <- df[df$Time != -1, ]
# df$Test[df$Test == "And_Future"] <- TeX("X")
# df$Test[df$Test == "And_Future"] <- "A&F(B)"
# df$Test[df$Test == "And_Globally"] <- "A&G(B)"
Expand All @@ -53,17 +54,19 @@ df$Max_Trace_Length <- as.character(df$Max_Trace_Length)

labels = c("AND", "OR")
names(labels) <- c("And", "Or")
print_chart(df[df$Test == "And" | df$Test == "Or",], labels, c(TeX(r'($OPTIMIZED$)'), TeX(r'($LOGICAL$)')))
print_chart(df[df$Test == "And" | df$Test == "Or",], labels, c(TeX(r'($OPTIMIZED$)'), TeX(r'($LOGICAL$)')), 'log10')

labels = c("Choice")
labels = c("CHOICE")
names(labels) <- c("Choice")
print_chart(df[df$Test == "Choice",], labels, c(TeX(r'($OPTIMIZED$)'), TeX(r'($LOGICAL$)')))
print_chart(df[df$Test == "Choice",], labels, c(TeX(r'($OPTIMIZED$)'), TeX(r'($LOGICAL$)')), 'log10')

labels = c("TIMED AND FUTURE", "TIMED AND GLOBALLY", "UNTIL")
names(labels) <- c("And_Future", "And_Globally", "Untimed Until")
print_chart(df[df$Test == "And_Future" | df$Test == "And_Globally" ,], labels, c(TeX(r'($VARIANT-2$)'), TeX(r'($VARIANT-1$)'), TeX(r'($LOGICAL$)')))
labels = c("TIMED AND FUTURE", "TIMED AND GLOBALLY")
names(labels) <- c("And_Future", "And_Globally")
print_chart(df[df$Test == "And_Future" | df$Test == "And_Globally" ,], labels, c(TeX(r'($VARIANT-1$)'), TeX(r'($VARIANT-2$)'), TeX(r'($LOGICAL$)')), 'log10')

print_chart(df[df$Test == "Until" ,], labels, c( TeX(r'($VARIANT-1$)'), TeX(r'($VARIANT-2$)')))
labels = c("UNTIL")
names(labels) <- c("Until")
print_chart(df[df$Test == "Until" ,], labels, c(TeX(r'($OPTIMIZED$)'), TeX(r'($LOGICAL$)')), 'log10')

# ggplot(df, aes(x=Log_Size, y=Time, fill=interaction(Max_Trace_Length,Type))) +
# geom_bar(position = "dodge", stat = "identity") +
Expand Down
5 changes: 5 additions & 0 deletions data/benchmarking_scripts/custom_operators.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/bin/bash
cd ..
cd ..

./cmake-build-release/benchmarking
7 changes: 7 additions & 0 deletions data/benchmarking_scripts/master_benchmarker.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
#!/bin/bash
./custom_operators.sh
./sqlminer.sh
./parallelization_pipeline.sh
./parallelization.sh
./phi.sh
./burattin.sh
Loading

0 comments on commit 53cd566

Please sign in to comment.