Fix minor things in scripts

generate_figs.py

@@ -300,7 +300,7 @@ def generate_figure_four(df):
 
     bench_list = sorted(list(set(list(df['bench']))))
 
-    item_no = len(bench_list)
+    item_no = 11
     item_per_row = 3
     x_dim = item_no // item_per_row + 1
     y_dim = min(item_no, item_per_row)
@@ -404,7 +404,7 @@ def generate_figure_five(df):
         'intersection',
         'union'
     ]
-    item_no = len(op_list)
+    item_no = 7
     x_dim = item_no // 3 + 1
     y_dim = min(item_no, 3)
 
@@ -635,13 +635,15 @@ def compute_relative(mata_d, other_d):
 
 
 def to_relative_table(df, rows, aggregation, tablefmt='latex'):
-    tools = ['mata', 'mata-0.111', 'mata-0.112', 'mata-0.113', 'mata-old', 'mata-sim', 'awali', 'mona', 'vata',  'automata.net', 'brics', 'automatalib-old', 'automatalib', 'fado', 'automata.py', '(py)mata']
+    tools = [
+        'mata', 'mata-sim', 'awali', 'mona', 'vata',  'automata.net', 'brics', 'automatalib', 'fado', 'automata.py', '(py)mata'
+    ]
     tools = [t for t in tools if t in set(df['tool'])]
     data = {
         grp: [grp] + ['-' for i in range(len(tools))] for grp in set(df[rows]) if grp != 'result' and 'result' not in grp
     }
     for grp, series in df.groupby([rows, 'tool'] if not isinstance(rows, list) else rows + ['tool']):
-        if grp[0] == 'result' or 'result' in grp[0]:
+        if grp[0] == 'result' or 'result' in grp[0] or grp[1] not in tools:
             continue
         vals = aggregation(series[['time', 'input']])
         data[grp[0]][tools.index(grp[1]) + 1] = vals

run_all.sh

@@ -9,6 +9,7 @@ OUTPUT_DIR_NAME=""
 METHODS=""
 EXCLUDE=""
 TEST_RUN=""
+TEST_RUN_COUNT=1
 SAY_YES="false"
 
 # Each benchmark corresponds to naming in our paper; set to "true" to run concrete benchmark
@@ -31,6 +32,7 @@ usage() { {
         echo "usage: ./run_all.sh [opt1, ..., optn]"
         echo "options:"
         echo "     --test-run               will conduct a test run (5s timeout, only 1 instance per benchmark)"
+        echo "     --test-run-count         will conduct a test run (5s timeout, only <INT> instance per benchmark)"
         echo "  -t|--timeout                will set a timeout to <INT> seconds (default 60s)"
         echo "  -o|--output-dir             will store the results in 'results/data/<DIR>'"
         echo "  -m|--methods                will measure only selected tools"
@@ -60,6 +62,10 @@ while [ $# -gt 0 ]; do
             TIMEOUT=5
             TEST_RUN="--test-run"
             shift 1;;
+        --test-run-count)
+            TIMEOUT=5
+            TEST_RUN="--test-run --test-run-count $2"
+            shift 2;;
         -y|--no-interaction)
             SAY_YES="true"
             shift 1;;

run_smoke_test.sh

@@ -1,2 +1,3 @@
 #!/bin/bash
-./run_all.sh --test-run --jobs 1 --b-smt --timeout 60
+rm -rf ./results/data/smoke-test || true
+./run_all.sh --test-run --jobs 1 --b-smt --timeout 60 --output-dir smoke-test --test-run-count 2

scripts/run_pyco.sh

@@ -18,6 +18,7 @@ usage() { {
         echo "  -s|--suffix                 adds suffix to the target file"
         echo "  -v|--verbose                adds verbosity to pycobench"
         echo "  -d|--test-run               will run only single benchmark from each input"
+        echo "  -c|--test-run-count         will run only <INT> benchmark from each input (used for test-run)"
     } >&2
 }
 
@@ -40,6 +41,7 @@ rootdir=$(realpath "$basedir/..")
 config="$rootdir/jobs/tacas-24-automata-inclusion.yaml"
 verbose=""
 testrun=false
+testrun_count=1
 GR='\033[0;32m'
 RD='\033[0;31m'
 NC='\033[0m' # No Color
@@ -73,6 +75,9 @@ while [ $# -gt 0 ]; do
         -d|--test-run)
             testrun=true
             shift 1;;
+        -c|--test-run-count)
+            testrun_count=$2
+            shift 2;;
         *)
             benchmarks+=( $1 )
             shift 1;;
@@ -138,7 +143,7 @@ do
     sub_result_file="$result_file.log"
     intermediate+=( $sub_result_file )
     if [ "$testrun" = true ]; then
-      cat "$benchmark_file" | head -1 | "$basedir/"pycobench $exclude $methods $verbose -j "$jobs" -c "$config" -t "$timeout" -o "$sub_result_file"
+      cat "$benchmark_file" | head -$testrun_count | "$basedir/"pycobench $exclude $methods $verbose -j "$jobs" -c "$config" -t "$timeout" -o "$sub_result_file"
     else
       "$basedir/"pycobench $exclude $methods $verbose -j "$jobs" -c "$config" -t "$timeout" -o "$sub_result_file" < "$benchmark_file"
     fi