Add mergiraf to the evaluated tools (#380)

.github/workflows/check-reproducibility-inconsistent-results.yml

@@ -37,6 +37,11 @@ jobs:
         channels: conda-forge,defaults
         activate-environment: AST
         environment-file: environment.yml
+    - name: Install Rust and Cargo
+      run: rustup update stable && rustup default stable
+    - name: Install Mergiraf
+      run: cargo install --locked mergiraf --version 0.4.0
+    - run: cargo --version
     - name: Clean caches & workspace
       run: make clean
     - run: echo "LOGURU_COLORIZE=NO" >> $GITHUB_ENV

.github/workflows/check-reproducibility.yml

@@ -25,6 +25,11 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
+    - name: Install Rust and Cargo
+      run: rustup update stable && rustup default stable
+    - run: cargo --version
+    - name: Install Mergiraf
+      run: cargo install --locked mergiraf --version 0.4.0
     - name: Set up Python 3.12
       uses: actions/setup-python@v5
       with:

.github/workflows/small-test.yml

@@ -37,6 +37,9 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
+    - name: Install Rust and Cargo
+      run: rustup update stable && rustup default stable
+    - run: cargo --version
     - name: Set up Python 3.12
       uses: actions/setup-python@v5
       with:

.gitignore

@@ -27,14 +27,15 @@ merge_repo/
 merges_small_valid_subsamples/
 repos/
 results/small/
-.workdir/
+.workdir*
 scratch/
 input_data/repos.csv.gz
 .pytest_cache/
 cf/
 .gradle/
 cache-machines/
 .old_cache
+wget*
 
 # Test output
 test/tmp1.txt

.pre-commit-config.yaml

@@ -9,9 +9,7 @@ repos:
     -   id: check-builtin-literals
     -   id: check-docstring-first
     -   id: check-executables-have-shebangs
-    -   id: check-json
     -   id: check-shebang-scripts-are-executable
-    -   id: pretty-format-json
     -   id: check-merge-conflict
     -   id: check-symlinks
     -   id: check-toml

Makefile

@@ -13,8 +13,6 @@ CSV_RESULTS_GREATEST_HITS = results/greatest_hits/result_raw.csv
 CSV_RESULTS_REAPER = results/reaper/result_raw.csv
 CSV_RESULTS = $(CSV_RESULTS_COMBINED)
 
-NUM_PROCESSES = 0
-
 showvars:
 	@echo "SH_SCRIPTS=${SH_SCRIPTS}"
 	@echo "BASH_SCRIPTS=${BASH_SCRIPTS}"
@@ -38,10 +36,6 @@ fix-java-style:
 check-java-style:
 	./gradlew -q spotlessCheck javadoc requireJavadoc -g ../.gradle/
 
-update-small-results:
-	rm -rf test/small-goal-files/
-	rsync -av --exclude='*.pdf' --exclude='*.png' --exclude='*unhandled_and_failed_merges_without_intellimerge*' --exclude='*.pgf' results/small/ test/small-goal-files/
-
 # This target deletes files that are not committed to version control.
 clean:
 	${MAKE} clean-workdir
@@ -112,16 +106,18 @@ copy-paper:
 # As of 2023-07-31, this takes 5-20 minutes to run, depending on your machine.
 small-test:
 	${MAKE} clean-test-cache clean
-	AST_REPOS_PATH=repos-small-test ./run_small.sh --include_trivial_merges --no_timing
+	AST_REPOS_PATH=repos-small-test TESTING=True DELETE_WORKDIRS=False WORKDIR_DIRECTORY=.workdir-small-test ./run_small.sh --include_trivial_merges --no_timing
 	${MAKE} compress-small-cache
 	${MAKE} small-test-diff
+	mv .workdir-small-test .workdir-small-test-temp
 	rm -rf results/small
-	AST_REPOS_PATH=repos-small-test ./run_small.sh --include_trivial_merges --no_timing
+	AST_REPOS_PATH=repos-small-test TESTING=True DELETE_WORKDIRS=False WORKDIR_DIRECTORY=.workdir-small-test ./run_small.sh --include_trivial_merges --no_timing
 	${MAKE} small-test-diff
+	mv .workdir-small-test-temp .workdir-small-test
 
 small-test-without-cleaning:
 	${MAKE} clean-test-cache
-	AST_REPOS_PATH=repos-small-test ./run_small.sh --include_trivial_merges --no_timing
+	AST_REPOS_PATH=repos-small-test TESTING=True DELETE_WORKDIRS=False WORKDIR_DIRECTORY=.workdir-small-test ./run_small.sh --include_trivial_merges --no_timing
 	${MAKE} small-test-diff
 
 update-figures:
@@ -130,7 +126,11 @@ update-figures:
 	./run_reaper.sh -op --no_timing
 
 update-figures-small:
-	AST_REPOS_PATH=repos-small-test ./run_small.sh -op --no_timing
+	AST_REPOS_PATH=repos-small-test DELETE_WORKDIRS=False WORKDIR_DIRECTORY=.workdir-small-test ./run_small.sh -op --no_timing
+
+update-small-results:
+	find test/small-goal-files/ -mindepth 1 -not -path "test/small-goal-files/hashes" -not -path "test/small-goal-files/hashes/*" -exec rm -rf {} +
+	rsync -av --exclude='*.pdf' --exclude='*.png' --exclude='*unhandled_and_failed_merges_without_intellimerge*' --exclude='*.pgf' results/small/ test/small-goal-files/
 
 run-all-without-timing:
 	${MAKE} clean-workdir
@@ -161,6 +161,8 @@ gradle-assemble:
 clean-workdir:
 	if [ -d .workdir ]; then chmod -R u+w .workdir; fi
 	rm -rf .workdir
+	if [ -d .workdir-small-test ]; then chmod -R u+w .workdir-small-test; fi
+	rm -rf .workdir-small-test
 
 clean-local:
 	${MAKE} clean-workdir

results/combined/defs.tex

@@ -12,20 +12,20 @@
 \def\combinedMergesSampled{6035\xspace}
 \def\combinedReposYieldedFull{1\xspace}
 \def\combinedReposTotal{1116\xspace}
-\def\combinedMergesTotal{5983\xspace}
+\def\combinedMergesTotal{5971\xspace}
 
 % Results
-\def\combinedAverageTriesUntilPass{1.0138133934137565\xspace}
-\def\combinedNumberofMergesWith1TriesUntilPass{90637\xspace}
-\def\combinedNumberofMergesWith3TriesUntilPass{167\xspace}
-\def\combinedNumberofMergesWith5TriesUntilPass{70\xspace}
-\def\combinedNumberofMergesWith2TriesUntilPass{514\xspace}
-\def\combinedNumberofMergesWith4TriesUntilPass{45\xspace}
-\def\combinedSporkOverOrtCorrect{512\xspace}
+\def\combinedAverageTriesUntilPass{1.0139492292821894\xspace}
+\def\combinedNumberofMergesWith1TriesUntilPass{97415\xspace}
+\def\combinedNumberofMergesWith3TriesUntilPass{180\xspace}
+\def\combinedNumberofMergesWith5TriesUntilPass{75\xspace}
+\def\combinedNumberofMergesWith2TriesUntilPass{567\xspace}
+\def\combinedNumberofMergesWith4TriesUntilPass{48\xspace}
+\def\combinedSporkOverOrtCorrect{509\xspace}
 \def\combinedSporkOverOrtIncorrect{486\xspace}
-\def\combinedMainBranchMerges{3524\xspace}
+\def\combinedMainBranchMerges{3515\xspace}
 \def\combinedMainBranchMergesPercent{59\xspace}
-\def\combinedOtherBranchMerges{2459\xspace}
+\def\combinedOtherBranchMerges{2456\xspace}
 \def\combinedOtherBranchMergesPercent{41\xspace}
 \def\combinedReposJava{42092\xspace}
 

results/combined/merges_tested/9apps/elasticsearch-cloudwatch.csv

@@ -1,2 +1,2 @@
-idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,left parent test result,left_tree_fingerprint,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,parents pass,right parent test result,right_tree_fingerprint,test merge,union_diff_files,sampled for testing,gitmerge_ort,gitmerge_ort_merge_fingerprint,gitmerge_ort_ignorespace,gitmerge_ort_ignorespace_merge_fingerprint,gitmerge_recursive_histogram,gitmerge_recursive_histogram_merge_fingerprint,gitmerge_recursive_myers_ignorespace,gitmerge_recursive_myers_ignorespace_merge_fingerprint,gitmerge_recursive_minimal,gitmerge_recursive_minimal_merge_fingerprint,gitmerge_recursive_myers,gitmerge_recursive_myers_merge_fingerprint,gitmerge_recursive_patience,gitmerge_recursive_patience_merge_fingerprint,gitmerge_resolve,gitmerge_resolve_merge_fingerprint,git_hires_merge,git_hires_merge_merge_fingerprint,spork,spork_merge_fingerprint,intellimerge,intellimerge_merge_fingerprint,adjacent,adjacent_merge_fingerprint,imports,imports_merge_fingerprint,version_numbers,version_numbers_merge_fingerprint,ivn,ivn_merge_fingerprint,ivn_ignorespace,ivn_ignorespace_merge_fingerprint,gitmerge_ort_plus,gitmerge_ort_plus_merge_fingerprint,gitmerge_ort_ignorespace_plus,gitmerge_ort_ignorespace_plus_merge_fingerprint,gitmerge_recursive_histogram_plus,gitmerge_recursive_histogram_plus_merge_fingerprint,gitmerge_recursive_myers_ignorespace_plus,gitmerge_recursive_myers_ignorespace_plus_merge_fingerprint,gitmerge_recursive_minimal_plus,gitmerge_recursive_minimal_plus_merge_fingerprint,gitmerge_recursive_myers_plus,gitmerge_recursive_myers_plus_merge_fingerprint,gitmerge_recursive_patience_plus,gitmerge_recursive_patience_plus_merge_fingerprint,gitmerge_resolve_plus,gitmerge_resolve_plus_merge_fingerprint,git_hires_merge_plus,git_hires_merge_plus_merge_fingerprint,spork_plus,spork_plus_merge_fingerprint,intellimerge_plus,intellimerge_plus_merge_fingerprint,adjacent_plus,adjacent_plus_merge_fingerprint,imports_plus,imports_plus_merge_fingerprint,version_numbers_plus,version_numbers_plus_merge_fingerprint,ivn_plus,ivn_plus_merge_fingerprint,ivn_ignorespace_plus,ivn_ignorespace_plus_merge_fingerprint
-2,refs/heads/master,8e72563c34b534951407fc553222e2d2b2536b0a,21d45d6b78e3e7c6f749cd7a8dd64897cc34a0d4,5e338acd3c7d8fab4361594dfc084ad2a89f31eb,,True,False,Tests_passed,8a1c1183895492fa26e72e027ade5ffda13b00ae025bee8bc8ef0773c8125d4b,True,2,3,4,2,True,Tests_passed,6346c5a5c40e39265013e7eba7b2e41781f1a17fe78679fdc734e4e48a403ff6,True,2,True,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,ab987682137551e396c32439051d9f6a1bbafa70346840447d9fd964ee150b80,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,58a2fb9f18ae675553af6c6a44bf85dc546ddb75efd21c37f64e81cdcfe6f0cb,Tests_failed,01e22091a4fb98a78e168e3b72fc85257d2b5c77966e628a15d89c2ff7741b3e,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02
+idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,left parent test result,left_tree_fingerprint,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,parents pass,right parent test result,right_tree_fingerprint,test merge,union_diff_files,sampled for testing,gitmerge_ort,gitmerge_ort_merge_fingerprint,gitmerge_ort_ignorespace,gitmerge_ort_ignorespace_merge_fingerprint,gitmerge_recursive_histogram,gitmerge_recursive_histogram_merge_fingerprint,gitmerge_recursive_myers_ignorespace,gitmerge_recursive_myers_ignorespace_merge_fingerprint,gitmerge_recursive_minimal,gitmerge_recursive_minimal_merge_fingerprint,gitmerge_recursive_myers,gitmerge_recursive_myers_merge_fingerprint,gitmerge_recursive_patience,gitmerge_recursive_patience_merge_fingerprint,gitmerge_resolve,gitmerge_resolve_merge_fingerprint,git_hires_merge,git_hires_merge_merge_fingerprint,spork,spork_merge_fingerprint,mergiraf,mergiraf_merge_fingerprint,intellimerge,intellimerge_merge_fingerprint,adjacent,adjacent_merge_fingerprint,imports,imports_merge_fingerprint,version_numbers,version_numbers_merge_fingerprint,ivn,ivn_merge_fingerprint,ivn_ignorespace,ivn_ignorespace_merge_fingerprint,gitmerge_ort_plus,gitmerge_ort_plus_merge_fingerprint,gitmerge_ort_ignorespace_plus,gitmerge_ort_ignorespace_plus_merge_fingerprint,gitmerge_recursive_histogram_plus,gitmerge_recursive_histogram_plus_merge_fingerprint,gitmerge_recursive_myers_ignorespace_plus,gitmerge_recursive_myers_ignorespace_plus_merge_fingerprint,gitmerge_recursive_minimal_plus,gitmerge_recursive_minimal_plus_merge_fingerprint,gitmerge_recursive_myers_plus,gitmerge_recursive_myers_plus_merge_fingerprint,gitmerge_recursive_patience_plus,gitmerge_recursive_patience_plus_merge_fingerprint,gitmerge_resolve_plus,gitmerge_resolve_plus_merge_fingerprint,git_hires_merge_plus,git_hires_merge_plus_merge_fingerprint,spork_plus,spork_plus_merge_fingerprint,mergiraf_plus,mergiraf_plus_merge_fingerprint,intellimerge_plus,intellimerge_plus_merge_fingerprint,adjacent_plus,adjacent_plus_merge_fingerprint,imports_plus,imports_plus_merge_fingerprint,version_numbers_plus,version_numbers_plus_merge_fingerprint,ivn_plus,ivn_plus_merge_fingerprint,ivn_ignorespace_plus,ivn_ignorespace_plus_merge_fingerprint
+2,refs/heads/master,8e72563c34b534951407fc553222e2d2b2536b0a,21d45d6b78e3e7c6f749cd7a8dd64897cc34a0d4,5e338acd3c7d8fab4361594dfc084ad2a89f31eb,,True,False,Tests_passed,8a1c1183895492fa26e72e027ade5ffda13b00ae025bee8bc8ef0773c8125d4b,True,2,3,4,2,True,Tests_passed,6346c5a5c40e39265013e7eba7b2e41781f1a17fe78679fdc734e4e48a403ff6,True,2,True,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,ab987682137551e396c32439051d9f6a1bbafa70346840447d9fd964ee150b80,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,58a2fb9f18ae675553af6c6a44bf85dc546ddb75efd21c37f64e81cdcfe6f0cb,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_failed,01e22091a4fb98a78e168e3b72fc85257d2b5c77966e628a15d89c2ff7741b3e,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02,Tests_passed,6a4a593297d55760b59fd05d342298f2ba803939c4b3a8448670ba5e2905fc02