Use hermit to provide a (mostly) consistent build environment (#700)

square · Nov 12, 2024 · cb935ca · cb935ca
1 parent 1e7f79a
commit cb935ca
Show file tree

Hide file tree

Showing 76 changed files with 561 additions and 682 deletions.
diff --git a/.github/workflows/binary-static-analysis.yml b/.github/workflows/binary-static-analysis.yml
@@ -21,8 +21,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [3.9]
 
     steps:
     - name: Checkout repository
@@ -33,21 +31,25 @@ jobs:
         fetch-depth: 2
         submodules: 'recursive'
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+    - name: Init Hermit
+      uses: cashapp/activate-hermit@v1
 
-    - name: Display Python version
+    - name: Install Clang
       run: |
-        python -c "import sys; print(sys.version)"
+        sudo apt-get update
+        sudo apt install clang -y
 
-    - name: Install build dependencies
+    - name: Install python packages
       run: |
-        scripts/ubuntu_install_protobuf.sh
+        cd ${{ github.workspace }}
+        pip install --upgrade pip
+        pip install -r ./core/requirements.txt
 
     - name: Build subzero CORE
       run: |
+        cd ${{ github.workspace }}
+        export CC=`which clang`
+        export CXX=`which clang++`
         scripts/build_core.sh
 
     # Run binary static analysis

diff --git a/.github/workflows/clang-static-analysis.yml b/.github/workflows/clang-static-analysis.yml
@@ -21,8 +21,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [3.9]
 
     steps:
     - name: Checkout repository
@@ -33,23 +31,27 @@ jobs:
         fetch-depth: 2
         submodules: 'recursive'
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+    - name: Init Hermit
+      uses: cashapp/activate-hermit@v1
 
-    - name: Display Python version
+    - name: Install clang and clang-tools
       run: |
-        python -c "import sys; print(sys.version)"
+        sudo apt-get update
+        sudo apt install clang clang-tools -y
 
-    - name: Install build dependencies
-      run: scripts/ubuntu_install_protobuf.sh
+    - name: Install python packages
+      run: |
+        cd ${{ github.workspace }}
+        pip install --upgrade pip
+        pip install -r ./core/requirements.txt
 
-    - name: Install clang tools
+    # Build CORE
+    - name: Build Subzero Core
       run: |
-        sudo apt-get update
-        sudo apt-get install clang-tools-14 -y
-        sudo ln -sf /usr/bin/clang-check-14 /usr/bin/clang-check
+        cd ${{ github.workspace }}
+        export CC=`which clang`
+        export CXX=`which clang++`
+        ./scripts/build_core.sh
 
     # Run static analysis
     - name: Run clang static analysis

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -25,7 +25,7 @@ jobs:
       fail-fast: false
       matrix:
         language: ['cpp', 'java']
-        python-version: [3.9]
+        python-version: [3.12]
 
     steps:
     - name: Checkout repository
@@ -36,18 +36,21 @@ jobs:
         fetch-depth: 2
         submodules: 'recursive'
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+    - name: Init Hermit
+      uses: cashapp/activate-hermit@v1
 
-    - name: Display Python version
+    - if: ${{ matrix.language == 'cpp' }}
+      name: Install Clang
       run: |
-        python -c "import sys; print(sys.version)"
+        sudo apt-get update
+        sudo apt install clang -y
 
     - if: ${{ matrix.language == 'cpp' }}
-      name: Install dependencies
-      run: scripts/ubuntu_install_protobuf.sh
+      name: Install python packages
+      run: |
+        cd ${{ github.workspace }}
+        pip install --upgrade pip
+        pip install -r ./core/requirements.txt
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -67,7 +70,11 @@ jobs:
     # Use custom build process for C/C++
     - if: ${{ matrix.language == 'cpp' }}
       name: Build Subzero Core (C/C++)
-      run: scripts/build_core.sh
+      run: |
+        cd ${{ github.workspace }}
+        export CC=`which clang`
+        export CXX=`which clang++`
+        ./scripts/build_core.sh
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/fuzzing-regression-test.yml b/.github/workflows/fuzzing-regression-test.yml
@@ -21,8 +21,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [3.9]
 
     steps:
     - name: Checkout repository
@@ -33,30 +31,27 @@ jobs:
         fetch-depth: 2
         submodules: 'recursive'
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
+    - name: Init Hermit
+      uses: cashapp/activate-hermit@v1
 
-    - name: Display Python version
+    - name: Install Clang
       run: |
-        python -c "import sys; print(sys.version)"
-
-    - name: Install dependencies
-      run: scripts/ubuntu_install_protobuf.sh
+        sudo apt-get update
+        sudo apt install clang -y
 
-    - name: Setup Clang
-      uses: egor-tensin/setup-clang@v1.4
-      with:
-        version: latest
-        platform: x64
+    - name: Install python packages
+      run: |
+        cd ${{ github.workspace }}
+        pip install --upgrade pip
+        pip install -r ./core/requirements.txt
 
     # Build CORE with ASAN, UBSAN, and libfuzzer
     - name: Build Subzero Core (ASAN+UBSAN+FUZZER)
       run: |
+        cd ${{ github.workspace }}
         export CC=`which clang`
         export CXX=`which clang++`
-        scripts/build_core.sh -DENABLE_ASAN=ON -DENABLE_UBSAN=ON -DENABLE_FUZZER=ON
+        ./scripts/build_core.sh -DENABLE_ASAN=ON -DENABLE_UBSAN=ON -DENABLE_FUZZER=ON
 
     - name: Run Subzero CORE Fuzzing regression test
       run: |

diff --git a/.github/workflows/java-test.yml b/.github/workflows/java-test.yml
@@ -21,8 +21,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [3.9]
 
     steps:
     - name: Checkout repository
@@ -33,11 +31,11 @@ jobs:
         fetch-depth: 2
         submodules: 'recursive'
 
-    - name: Setup Gradle
-      uses: gradle/gradle-build-action@v2
+    - name: Init Hermit
+      uses: cashapp/activate-hermit@v1
 
     # Unit tests for Java
     - name: Run Unit tests
       run: |
         cd ${{ github.workspace }}/java
-        ./gradlew test --info
+        gradle test --info
diff --git a/.github/workflows/signtx-test-asan.yml b/.github/workflows/signtx-test-asan.yml
@@ -21,8 +21,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [3.9]
 
     steps:
     - name: Checkout repository
@@ -33,30 +31,33 @@ jobs:
         fetch-depth: 2
         submodules: 'recursive'
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Display Python version
-      run: |
-        python -c "import sys; print(sys.version)"
-
-    - name: Setup Gradle
-      uses: gradle/gradle-build-action@v2
-
-    - name: Install dependencies
-      run: scripts/ubuntu_install_protobuf.sh
+    - name: Init Hermit
+      uses: cashapp/activate-hermit@v1
 
     # Build Java targets
     - name: Build Java
       run: |
         cd ${{ github.workspace }}/java
-        ./gradlew clean build --info
+        gradle build --info
+
+    - name: Install Clang
+      run: |
+        sudo apt-get update
+        sudo apt install clang -y
+
+    - name: Install python packages
+      run: |
+        cd ${{ github.workspace }}
+        pip install --upgrade pip
+        pip install -r ./core/requirements.txt
 
     # Build CORE with ASAN
     - name: Build Subzero Core with ASAN
-      run: scripts/build_core.sh -DENABLE_ASAN=ON
+      run: |
+        cd ${{ github.workspace }}
+        export CC=`which clang`
+        export CXX=`which clang++`
+        ./scripts/build_core.sh -DENABLE_ASAN=ON
 
     - name: Run Subzero CORE with ASAN
       run: |

diff --git a/.github/workflows/signtx-test-msan.yml b/.github/workflows/signtx-test-msan.yml
@@ -21,8 +21,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      matrix:
-        python-version: [3.9]
 
     steps:
     - name: Checkout repository
@@ -33,39 +31,33 @@ jobs:
         fetch-depth: 2
         submodules: 'recursive'
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Display Python version
-      run: |
-        python -c "import sys; print(sys.version)"
-
-    - name: Setup Gradle
-      uses: gradle/gradle-build-action@v2
-
-    - name: Install dependencies
-      run: scripts/ubuntu_install_protobuf.sh
+    - name: Init Hermit
+      uses: cashapp/activate-hermit@v1
 
     # Build Java targets
     - name: Build Java
       run: |
         cd ${{ github.workspace }}/java
-        ./gradlew clean build --info
+        gradle build --info
 
-    - name: Setup Clang (for MSAN)
-      uses: egor-tensin/setup-clang@v1.4
-      with:
-        version: latest
-        platform: x64
+    - name: Install Clang
+      run: |
+        sudo apt-get update
+        sudo apt install clang -y
+
+    - name: Install python packages
+      run: |
+        cd ${{ github.workspace }}
+        pip install --upgrade pip
+        pip install -r ./core/requirements.txt
 
     # Build CORE with MSAN
     - name: Build Subzero Core with MSAN
       run: |
+        cd ${{ github.workspace }}
         export CC=`which clang`
         export CXX=`which clang++`
-        scripts/build_core.sh -DENABLE_MSAN=ON
+        ./scripts/build_core.sh -DENABLE_MSAN=ON
 
     - name: Run Subzero CORE with MSAN
       run: |