Added vanilla analisys

script-python/journal/README.md

@@ -8,6 +8,7 @@ solc-select use 0.5.17
 pip install tqdm
 pip install scipy
 git clone https://github.com/DependableSystemsLab/SolidiFI-benchmark
+git clone https://github.com/DependableSystemsLab/SolidiFI.git
 ```
 
 Compile bytecodes:

script-python/journal/compile-all.py

@@ -2,10 +2,6 @@
 import subprocess
 import json
 
-source_dir = './reentrancy/source-code'
-json_dir = './reentrancy/json'
-bytecode_dir = './reentrancy/bytecode'
-
 def clear_directory(directory):
     """
     Clears all files in the specified directory.
@@ -18,7 +14,7 @@ def clear_directory(directory):
             except Exception as e:
                 print(f"Error deleting {file_path}: {e}")
 
-def compile_solidity_sources():
+def compile_solidity_sources(source_dir, json_dir):
     """
     Compiles all .sol files in the specified source directory using solc, 
     saving the bytecode for each file in JSON format in the specified output directory.
@@ -44,7 +40,7 @@ def compile_solidity_sources():
             except subprocess.CalledProcessError as e:
                 print(f"Error compiling {filename}: {e}")
 
-def extract_and_save_longest_bytecode():
+def extract_and_save_longest_bytecode(bytecode_dir, json_dir):
     """
     Extracts the longest bytecode from each .json file and saves it in the specified output directory.
     """
@@ -87,7 +83,7 @@ def extract_and_save_longest_bytecode():
                         bytecode_file.write("0x" + longest_bytecode)
                     # print(f"Extracted longest bytecode from {longest_contract_name} to {bytecode_filename}")
 
-def extract_and_save_bytecode():
+def extract_and_save_bytecode(bytecode_dir, json_dir):
     """
     Extracts all bytecode from each .json file and saves it in the specified output directory.
     """
@@ -117,6 +113,18 @@ def extract_and_save_bytecode():
                         count += 1  # Increment counter for next bytecode
 
 if __name__ == "__main__":
-    compile_solidity_sources()
-    # extract_and_save_bytecode()
-    extract_and_save_longest_bytecode()
+
+    compile_solidity_sources('./reentrancy/source-code',
+                             './reentrancy/json')
+    # extract_and_save_bytecode('./reentrancy/bytecode',
+    #                           './reentrancy/json')
+    extract_and_save_longest_bytecode('./reentrancy/bytecode',
+                                      './reentrancy/json')
+
+
+    compile_solidity_sources('./vanilla/source-code',
+                             './vanilla/json')
+    # extract_and_save_bytecode('./vanilla/bytecode',
+    #                           './vanilla/json')
+    extract_and_save_longest_bytecode('./vanilla/bytecode',
+                                      './vanilla/json')

script-python/journal/run-benchmark.py

@@ -11,10 +11,6 @@
 import json
 
 # Directory paths
-bytecode_dir = './reentrancy/bytecode'
-results_dir = './reentrancy/results'
-result_evmlisa_dir = results_dir + '/evmlisa'
-result_ethersolve_dir = results_dir + '/ethersolve'
 max_threads = int(os.cpu_count() / 3)  # Core avaiable
 
 #################################### Utility
@@ -78,6 +74,18 @@ def plot_results(data_evmlisa, data_ethersolve, data_solidifi):
 
     plt.show()
 
+def subtract_dicts(dict1, dict2):
+    result = {}
+
+    for key in dict1:
+        if key in dict2:
+            result[key] = dict1[key] - dict2[key]
+        else:
+            result[key] = dict1[key]
+
+    return result
+
+
 #################################### EVMLiSA
 
 def build_evmlisa():
@@ -91,7 +99,7 @@ def build_evmlisa():
     subprocess.run(command, shell=True, check=True)
     print("[EVMLISA] EVMLiSA built successfully.")
 
-def run_evmlisa(bytecode_file):
+def run_evmlisa(bytecode_file, result_evmlisa_dir):
     """
     Runs the EVMLiSA analysis for a given bytecode file.
     
@@ -123,7 +131,7 @@ def run_evmlisa(bytecode_file):
         print(f"[EVMLISA] Error analyzing {bytecode_file}: {e}")
         return None
 
-def evmlisa():
+def evmlisa(bytecode_dir, results_dir, result_evmlisa_dir):
     """
     Main function to run EVMLiSA analyses on multiple bytecode files.
     """
@@ -143,7 +151,7 @@ def evmlisa():
 
     # Run analyses in parallel
     with ThreadPoolExecutor(max_workers=max_threads) as executor:
-        future_to_file = {executor.submit(run_evmlisa, file): file for file in bytecode_files}
+        future_to_file = {executor.submit(run_evmlisa, file, result_evmlisa_dir): file for file in bytecode_files}
 
         with tqdm(total=num_files, desc="[EVMLISA] Analyzing bytecode files") as pbar:
             for future in as_completed(future_to_file):
@@ -181,7 +189,7 @@ def check_sound_analysis_evmlisa(directory_path):
     if sound:
         print("[EVMLiSA] All analysis are SOUND")
 
-def results_evmlisa(directory_path):
+def results_evmlisa(directory_path, print_data):
     re_entrancy_warning_counts = {}
 
     for filename in os.listdir(directory_path):
@@ -190,7 +198,6 @@ def results_evmlisa(directory_path):
             try:
                 with open(file_path, 'r') as file:
                     data = json.load(file)
-
                     if "re-entrancy-warning" in data:
                         re_entrancy_warning_counts[filename] = data['re-entrancy-warning']
                     else:
@@ -209,15 +216,23 @@ def results_evmlisa(directory_path):
         if match:
             id = int(match.group(1))
             results[id] += result
+
+        match = re.match(r'(\d+)-\w+\.json', file)
+        if match:
+            id = int(match.group(1))
+            results[id] += result
 
     sorted_data = dict(sorted(results.items()))
+
+    print(print_data)
     print(sorted_data)
+
     return sorted_data
 
 
 #################################### EtherSolve
 
-def run_ethersolve(bytecode_file):
+def run_ethersolve(bytecode_file, result_ethersolve_dir):
     """
     Runs the EtherSolve analysis for a given bytecode file.
     
@@ -249,7 +264,7 @@ def run_ethersolve(bytecode_file):
         # print(f"[ETHERSOLVE] Error analyzing {bytecode_file}: {e}")
         return None
 
-def ethersolve():
+def ethersolve(bytecode_dir, result_ethersolve_dir):
     """
     Main function to run EtherSolve analyses on multiple bytecode files.
     """
@@ -266,7 +281,7 @@ def ethersolve():
 
     # Run analyses in parallel
     with ThreadPoolExecutor(max_workers=1) as executor:
-        future_to_file = {executor.submit(run_ethersolve, file): file for file in bytecode_files}
+        future_to_file = {executor.submit(run_ethersolve, file, result_ethersolve_dir): file for file in bytecode_files}
 
         with tqdm(total=num_files, desc="[ETHERSOLVE] Analyzing bytecode files") as pbar:
             for future in as_completed(future_to_file):
@@ -278,7 +293,7 @@ def ethersolve():
     print(f"[ETHERSOLVE] Completed {analysis_ended}/{num_files}.")
     delete_tmp_files(bytecode_dir)
 
-def results_ethersolve(directory_path):
+def results_ethersolve(directory_path, print_data):
     """
     Counts occurrences of the word "SSTORE" in files with "reentrancy" in their names 
     within the specified directory.
@@ -308,12 +323,15 @@ def results_ethersolve(directory_path):
             results[id] += result
 
     sorted_data = dict(sorted(results.items()))
-    # print(sorted_data)
+
+    print(print_data)
+    print(sorted_data)
+
     return sorted_data
 
 #################################### SolidiFI
 
-def results_solidifi(folder_path):
+def results_solidifi(folder_path, print_data):
     # Initialize a dictionary to store the line count for each problem ID
     line_counts = defaultdict(int)
 
@@ -334,27 +352,39 @@ def results_solidifi(folder_path):
             line_counts[problem_id] = num_lines - 1
 
     sorted_data = dict(sorted(line_counts.items()))
-    # print(sorted_data)
+
+    print(print_data)
+    print(sorted_data)
+
     return sorted_data
 
 #################################### Main
 
 if __name__ == "__main__":
-
-    evmlisa_thread = threading.Thread(target=evmlisa)
-    ethersolve_thread = threading.Thread(target=ethersolve)
+    evmlisa_vanilla_thread = threading.Thread(target=evmlisa, kwargs={'bytecode_dir':       './vanilla/bytecode', 
+                                                                      'results_dir':        './vanilla/results',
+                                                                      'result_evmlisa_dir': './vanilla/results/evmlisa'})
+    evmlisa_thread = threading.Thread(target=evmlisa, kwargs={'bytecode_dir':       './reentrancy/bytecode', 
+                                                              'results_dir':        './reentrancy/results',
+                                                              'result_evmlisa_dir': './reentrancy/results/evmlisa'})
+    ethersolve_thread = threading.Thread(target=ethersolve, kwargs={'bytecode_dir':             './reentrancy/bytecode',
+                                                                    'result_ethersolve_dir':    './reentrancy/results/ethersolve'})
 
+    evmlisa_vanilla_thread.start()
     evmlisa_thread.start()
     ethersolve_thread.start()
 
+    evmlisa_vanilla_thread.join()
     evmlisa_thread.join()
     ethersolve_thread.join()
 
-    check_sound_analysis_evmlisa(result_evmlisa_dir)
-
+    check_sound_analysis_evmlisa('./reentrancy/results/evmlisa')
+    check_sound_analysis_evmlisa('./vanilla/results/evmlisa')
+
     plot_results(
-        results_evmlisa(result_evmlisa_dir),
-        results_ethersolve(result_ethersolve_dir),
-        results_solidifi("./SolidiFI-benchmark/buggy_contracts/Re-entrancy")
+        subtract_dicts(     results_evmlisa('./reentrancy/results/evmlisa', 'evmlisa-buggy'),
+                            results_evmlisa('./vanilla/results/evmlisa', 'evmlisa-vanilla')),
+        results_ethersolve( './reentrancy/results/ethersolve', 'ethersolve'),
+        results_solidifi(   "./SolidiFI-benchmark/buggy_contracts/Re-entrancy", 'solidify')
     )
 

script-python/journal/vanilla/bytecode/1.bytecode

@@ -0,0 +1 @@
+0x608060405234801561001057600080fd5b50600436106100a95760003560e01c8063313ce56711610071578063313ce567146102935780635c658165146102b757806370a082311461032f57806395d89b4114610387578063a9059cbb1461040a578063dd62ed3e14610470576100a9565b806306fdde03146100ae578063095ea7b31461013157806318160ddd1461019757806323b872dd146101b557806327e235e31461023b575b600080fd5b6100b66104e8565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f65780820151818401526020810190506100db565b50505050905090810190601f1680156101235780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b61017d6004803603604081101561014757600080fd5b81019080803573ffffffffffffffffffffffffffffffffffffffff16906020019092919080359060200190929190505050610586565b604051808215151515815260200191505060405180910390f35b61019f610678565b6040518082815260200191505060405180910390f35b610221600480360360608110156101cb57600080fd5b81019080803573ffffffffffffffffffffffffffffffffffffffff169060200190929190803573ffffffffffffffffffffffffffffffffffffffff1690602001909291908035906020019092919050505061067e565b604051808215151515815260200191505060405180910390f35b61027d6004803603602081101561025157600080fd5b81019080803573ffffffffffffffffffffffffffffffffffffffff169060200190929190505050610916565b6040518082815260200191505060405180910390f35b61029b61092e565b604051808260ff1660ff16815260200191505060405180910390f35b610319600480360360408110156102cd57600080fd5b81019080803573ffffffffffffffffffffffffffffffffffffffff169060200190929190803573ffffffffffffffffffffffffffffffffffffffff169060200190929190505050610941565b6040518082815260200191505060405180910390f35b6103716004803603602081101561034557600080fd5b81019080803573ffffffffffffffffffffffffffffffffffffffff169060200190929190505050610966565b6040518082815260200191505060405180910390f35b61038f6109af565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156103cf5780820151818401526020810190506103b4565b50505050905090810190601f1680156103fc5780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b6104566004803603604081101561042057600080fd5b81019080803573ffffffffffffffffffffffffffffffffffffffff16906020019092919080359060200190929190505050610a4d565b604051808215151515815260200191505060405180910390f35b6104d26004803603604081101561048657600080fd5b81019080803573ffffffffffffffffffffffffffffffffffffffff169060200190929190803573ffffffffffffffffffffffffffffffffffffffff169060200190929190505050610ba4565b6040518082815260200191505060405180910390f35b60038054600181600116156101000203166002900480601f01602080910402602001604051908101604052809291908181526020018280546001816001161561010002031660029004801561057e5780601f106105535761010080835404028352916020019161057e565b820191906000526020600020905b81548152906001019060200180831161056157829003601f168201915b505050505081565b600081600260003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167f8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925846040518082815260200191505060405180910390a36001905092915050565b60005481565b600080600260008673ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002054905082600160008773ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020541015801561074f5750828110155b61075857600080fd5b82600160008673ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828254019250508190555082600160008773ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600082825403925050819055507fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8110156108a55782600260008773ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600082825403925050819055505b8373ffffffffffffffffffffffffffffffffffffffff168573ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef856040518082815260200191505060405180910390a360019150509392505050565b60016020528060005260406000206000915090505481565b600460009054906101000a900460ff1681565b6002602052816000526040600020602052806000526040600020600091509150505481565b6000600160008373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020549050919050565b60058054600181600116156101000203166002900480601f016020809104026020016040519081016040528092919081815260200182805460018160011615610100020316600290048015610a455780601f10610a1a57610100808354040283529160200191610a45565b820191906000526020600020905b815481529060010190602001808311610a2857829003601f168201915b505050505081565b600081600160003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020541015610a9b57600080fd5b81600160003373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828254039250508190555081600160008573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020600082825401925050819055508273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040518082815260200191505060405180910390a36001905092915050565b6000600260008473ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000205490509291505056fea265627a7a72315820d3426bdd231bdaaedacd04f0d235a6d884e7e093e97a4961f86bb7db70d64bec64736f6c63430005110032