treewide: Improve docker container build, CI caching and python depen…

…dencies (#192) Co-authored-by: Luca Colagrande <luca.colagrande3@gmail.com>
pulp-platform · Sep 27, 2024 · 379ae99 · 379ae99
1 parent e0fc33a
commit 379ae99
Show file tree

Hide file tree

Showing 16 changed files with 287 additions and 325 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,4 @@
+*
+!sw/**/*.py
+!util/**/*.py
+!pyproject.toml
diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,17 +5,55 @@
 # Run functional regression checks
 name: ci
 on: [push, pull_request]
+
 jobs:
 
+  ##########################
+  # Build Docker Container #
+  ##########################
+
+  build-docker:
+    name: Deploy Docker image
+    runs-on: ubuntu-22.04
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          docker-images: false
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - name: GHCR Log-in
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          cache-from: type=gha
+          cache-to: type=gha,mode=max`
+          file: util/container/Dockerfile
+          push: true
+          tags: ghcr.io/pulp-platform/snitch_cluster:${{ github.ref_name }}
+
   ########
   # Docs #
   ########
 
   docs:
     name: Build documentation
     runs-on: ubuntu-22.04
+    needs: build-docker
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:${{ github.ref_name }}
     steps:
       - uses: actions/checkout@v2
       - name: Build docs
@@ -28,8 +66,12 @@ jobs:
   pytest:
     name: Python unit tests
     runs-on: ubuntu-22.04
+    needs: build-docker
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:${{ github.ref_name }}
     steps:
       - uses: actions/checkout@v2
       - name: Run pytest
@@ -42,23 +84,40 @@ jobs:
   sw-snitch-cluster-vlt:
     name: Simulate SW on Snitch Cluster w/ Verilator
     runs-on: ubuntu-22.04
+    needs: build-docker
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:${{ github.ref_name }}
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
-      - name: Install local Python packages
-        run: pip install .
-      - name: Build Software
-        working-directory: target/snitch_cluster
-        run: |
-          bender vendor init
-          make CFG_OVERRIDE=cfg/github-ci.hjson sw
+      - name: Hash Verilator prerequisites
+        id: verilator-hash
+        uses: colluca/list-make-prerequisites@v1.0.1
+        with:
+          working-directory: target/snitch_cluster
+          target: bin/snitch_cluster.vlt
+          flags: --recursive
+      - name: Set up cache for Verilator build
+        id: verilator-cache
+        uses: actions/cache@v3
+        with:
+          path: target/snitch_cluster/bin
+          key: verilator-${{ steps.verilator-hash.outputs.hash }}
+          restore-keys: |
+            verilator-
       - name: Build Hardware
+        if: steps.verilator-cache.outputs.cache-hit != 'true'
         working-directory: target/snitch_cluster
         run: |
           make CFG_OVERRIDE=cfg/github-ci.hjson VLT_JOBS=1 bin/snitch_cluster.vlt
+      - name: Build Software
+        working-directory: target/snitch_cluster
+        run: |
+          make CFG_OVERRIDE=cfg/github-ci.hjson sw
       - name: Run Tests
         working-directory: target/snitch_cluster
         run: |
@@ -69,28 +128,64 @@ jobs:
         run: |
           make SIM_DIR=./runs/simple annotate -j
 
-  #########################################
-  # Build SW on Snitch Cluster w/ Banshee #
-  #########################################
+  # #########################################
+  # # Build SW on Snitch Cluster w/ Banshee #
+  # #########################################
+
+  # sw-snitch-cluster-banshee:
+  #   name: Simulate SW on Snitch Cluster w/ Banshee
+  #   runs-on: ubuntu-22.04
+  #   needs: build-docker
+  #   if: >
+  #     github.event_name != 'pull_request' ||
+  #     github.event.pull_request.head.repo.full_name != github.repository
+  #   container:
+  #     image: ghcr.io/pulp-platform/snitch_cluster:${{ github.ref_name }}
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #       with:
+  #         submodules: 'recursive'
+  #     - name: Build Software
+  #       run: |
+  #         bender vendor init
+  #         make -C target/snitch_cluster SELECT_RUNTIME=banshee sw
+  #     - name: Run Tests
+  #       env:
+  #         SNITCH_LOG: info
+  #       working-directory: target/snitch_cluster
+  #       run: |
+  #         ./util/run.py sw/run.yaml --simulator banshee -j
 
-  sw-snitch-cluster-banshee:
-    name: Simulate SW on Snitch Cluster w/ Banshee
+  ######################
+  # Sources Up-to-Date #
+  ######################
+  sources-up-to-date:
+    name: Check Sources Up-to-Date
     runs-on: ubuntu-22.04
+    needs: build-docker
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
+      image: ghcr.io/pulp-platform/snitch_cluster:${{ github.ref_name }}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
-          submodules: 'recursive'
-      - name: Install local Python packages
-        run: pip install .
-      - name: Build Software
+          submodules: true
+      - name: Generate opcodes
         run: |
-          bender vendor init
-          make -C target/snitch_cluster SELECT_RUNTIME=banshee sw
-      - name: Run Tests
-        env:
-          SNITCH_LOG: info
+          ./util/generate-opcodes.sh
+      - name: Generate RTL sources
         working-directory: target/snitch_cluster
         run: |
-          ./util/run.py sw/run.yaml --simulator banshee -j
+          make rtl
+      # For some reason, the checkout is done by a different user,
+      # than that running `git diff` (root, possibly due to Docker).
+      # So we need to set the repository as a safe directory.
+      - name: Git config safe.directory
+        run: |
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+      - name: Diff porcelain
+        uses: mmontes11/diff-porcelain@v0.0.1
+        with:
+          message: Found differences, please update all sources
diff --git a/.github/workflows/gitlab-ci.yaml b/.github/workflows/gitlab-ci.yaml
@@ -13,10 +13,10 @@ jobs:
       - name: Check Gitlab CI
         uses: pulp-platform/pulp-actions/gitlab-ci@v2.1.0
         # Skip on forks or pull requests from forks due to missing secrets.
-        if:
-          # yamllint disable rule:line-length
-          github.repository == 'pulp-platform/snitch_cluster' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
-          # yamllint enable rule:line-length
+        if: >
+          github.repository == 'pulp-platform/snitch_cluster' &&
+          (github.event_name != 'pull_request' ||
+          github.event.pull_request.head.repo.full_name == github.repository)
         with:
           domain: iis-git.ee.ethz.ch
           repo: github-mirror/snitch_cluster

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -14,6 +14,9 @@ jobs:
   verible-lint:
     name: Lint Verilog sources
     runs-on: ubuntu-latest
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@v3
       - uses: chipsalliance/verible-linter-action@main
@@ -32,48 +35,22 @@ jobs:
   bender-vendor-up-to-date:
     name: Check bender vendor up-to-date
     runs-on: ubuntu-latest
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Check bender vendor up-to-date
         uses: pulp-platform/pulp-actions/bender-vendor-up-to-date@v2.1.0
 
-  ######################
-  # Sources Up-to-Date #
-  ######################
-  sources-up-to-date:
-    name: Check Sources Up-to-Date
-    runs-on: ubuntu-22.04
-    container:
-      image: ghcr.io/pulp-platform/snitch_cluster:main
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-      - name: Install local Python packages
-        run: pip install .
-      - name: Generate opcodes
-        run: |
-          ./util/generate-opcodes.sh
-      - name: Generate RTL sources
-        working-directory: target/snitch_cluster
-        run: |
-          make rtl
-      # For some reason, the checkout is done by a different user,
-      # than that running `git diff` (root, possibly due to Docker).
-      # So we need to set the repository as a safe directory.
-      - name: Git config safe.directory
-        run: |
-          git config --global --add safe.directory $GITHUB_WORKSPACE
-      - name: Diff porcelain
-        uses: mmontes11/diff-porcelain@v0.0.1
-        with:
-          message: Found differences, please update all sources
-
   #################
   # Check License #
   #################
   license-lint:
     name: Check License headers
     runs-on: ubuntu-latest
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - name: Check License
         uses: pulp-platform/pulp-actions/lint-license@v2.1.0
@@ -97,6 +74,9 @@ jobs:
   yaml-lint:
     name: Lint YAML Sources
     runs-on: ubuntu-latest
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@v3
       - name: yaml-lint
@@ -109,6 +89,9 @@ jobs:
   ########################
   python-lint:
     runs-on: ubuntu-latest
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     name: Lint Python Sources
     steps:
       - name: Check out source repository
@@ -130,9 +113,12 @@ jobs:
   clangfmt:
     name: Lint C/C++ Sources
     runs-on: ubuntu-latest
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@v3
-      - uses: DoozyX/clang-format-lint-action@v0.16.2
+      - uses: DoozyX/clang-format-lint-action@v0.18.1
         with:
           clangFormatVersion: 10
 
@@ -143,6 +129,9 @@ jobs:
   editorconfig-lint:
     name: Lint Editorconfig
     runs-on: ubuntu-latest
+    if: >
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name != github.repository
     steps:
       - uses: actions/checkout@v3
       - uses: editorconfig-checker/action-editorconfig-checker@main

diff --git a/apt-requirements.txt b/apt-requirements.txt
diff --git a/docs/requirements.txt b/docs/requirements.txt