run-vmtest@v2 and build-selftests@v2

.github/workflows/kernel-build.yml

@@ -46,16 +46,13 @@ jobs:
     timeout-minutes: 100
     env:
         ARTIFACTS_ARCHIVE: "vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst"
-        BASE_BRANCH: >-
-          ${{ github.event_name == 'push' && github.ref_name
-              || github.base_ref
-              || 'bpf-next'
-          }}
         BUILD_SCHED_EXT_SELFTESTS: ${{ inputs.arch == 'x86_64' || inputs.arch == 'aarch64' && 'true' || '' }}
+        CACHED_KERNEL_BUILD: ${{ github.event_name == 'push' || github.repository == 'libbpf/ci'  && 'true' || '' }}
         KBUILD_OUTPUT: ${{ github.workspace }}/kbuild-output
         KERNEL: ${{ inputs.kernel }}
         REPO_PATH: ""
         REPO_ROOT: ${{ github.workspace }}
+        KERNEL_ROOT: ${{ github.workspace }}
     steps:
       - uses: actions/checkout@v4
         # We fetch an actual bit of history here to facilitate incremental
@@ -64,23 +61,32 @@ jobs:
           fetch-depth: 50
       - if: ${{ inputs.download_sources }}
         name: Download bpf-next tree
+        env:
+          FETCH_DEPTH: ${{ env.CACHED_KERNEL_BUILD && 16 || 0 }}
         uses: ./get-linux-source
         with:
           dest: '.kernel'
-      - if: ${{ inputs.download_sources }}
-        name: Move linux source in place
-        shell: bash
-        run: |
-          rm -rf .kernel/.git
-          cp -rf .kernel/. .
-          rm -rf .kernel
-      - uses: ./prepare-incremental-build
+      - if: ${{ env.CACHED_KERNEL_BUILD }}
+        uses: ./prepare-incremental-build
         with:
-          repo-root: ${{ env.REPO_ROOT }}
-          base-branch: ${{ env.BASE_BRANCH }}
+          repo-root: ${{ inputs.download_sources && '.kernel' || env.REPO_ROOT }}
+          base-branch: >-
+            ${{    github.repository == 'kernel-patches/bpf' && github.event_name == 'push' && github.ref_name
+                || github.repository == 'kernel-patches/bpf' && github.event_name != 'push' && github.base_ref
+                || 'master'
+             }}
           arch: ${{ inputs.arch }}
           toolchain_full: ${{ inputs.toolchain_full }}
           kbuild-output: ${{ env.KBUILD_OUTPUT }}
+      - if: ${{ inputs.download_sources }}
+        name: Move linux source in place
+        shell: bash
+        run: |
+          cd .kernel
+          rm -rf .git
+          mv -t .. $(ls -A)
+          cd ..
+          rmdir .kernel
       - uses:  ./patch-kernel
         with:
           patches-root: '${{ github.workspace }}/ci/diffs'
@@ -109,17 +115,15 @@ jobs:
           llvm-version: ${{ inputs.llvm-version }}
       - name: Build selftests/bpf
         uses: ./build-selftests
+        env:
+          MAX_MAKE_JOBS: 32
+          RELEASE: ${{ inputs.release && '1' || '' }}
         with:
           arch: ${{ inputs.arch }}
-          toolchain: ${{ inputs.toolchain }}
-          kbuild-output: ${{ env.KBUILD_OUTPUT }}
-          max-make-jobs: 32
+          kernel-root: ${{ env.KERNEL_ROOT }}
           llvm-version: ${{ inputs.llvm-version }}
-        env:
-          # RELEASE= disables all optimizaions
-          # RELEASE=0 adds -O0 make flag
-          # RELEASE=1 adds -O2 make flag
-          RELEASE: ${{ inputs.release && '1' || '' }}
+          toolchain: ${{ inputs.toolchain }}
+
       - if: ${{ env.BUILD_SCHED_EXT_SELFTESTS }}
         name: Build selftests/sched_ext
         uses: ./build-scx-selftests
@@ -151,7 +155,7 @@ jobs:
           archive: ${{ env.ARTIFACTS_ARCHIVE }}
           kbuild-output: ${{ env.KBUILD_OUTPUT }}
           repo-root: ${{ env.REPO_ROOT }}
-      - if: ${{ github.event_name != 'push' }}
+      - if: ${{ ! env.CACHED_KERNEL_BUILD }}
         name: Remove KBUILD_OUTPUT content
         shell: bash
         run: |

.github/workflows/kernel-test.yml

@@ -38,21 +38,46 @@ jobs:
     runs-on: ${{ fromJSON(inputs.runs_on) }}
     timeout-minutes: 100
     env:
+      ARCH: ${{ inputs.arch }}
       KERNEL: ${{ inputs.kernel }}
       REPO_ROOT: ${{ github.workspace }}
       REPO_PATH: ""
       # https://github.com/actions/runner/issues/1483#issuecomment-1031671517
       # booleans are weird in GH.
       CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }}
+      DEPLOYMENT: ${{ github.repository == 'kernel-patches/bpf' && 'prod' || 'rc' }}
+      ALLOWLIST_FILE: /tmp/allowlist
+      DENYLIST_FILE: /tmp/denylist
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/download-artifact@v4
         with:
           name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}
           path: .
+
       - name: Untar artifacts
         # zstd is installed by default in the runner images.
         run: zstd -d -T0  vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst --stdout | tar -xf -
+
+      - name: Prepare ALLOW/DENYLIST
+        env:
+          SELFTESTS_BPF: ${{ github.workspace }}/selftests/bpf
+          VMTEST_CONFIGS: ${{ github.workspace }}/ci/vmtest/configs
+        run: |
+          cat "${SELFTESTS_BPF}/ALLOWLIST"          \
+              "${SELFTESTS_BPF}/ALLOWLIST.${ARCH}"  \
+              "${VMTEST_CONFIGS}/ALLOWLIST"         \
+              "${VMTEST_CONFIGS}/ALLOWLIST.${ARCH}" \
+            2> /dev/null > "${ALLOWLIST_FILE}" || true
+
+          cat "${SELFTESTS_BPF}/DENYLIST" \
+              "${SELFTESTS_BPF}/DENYLIST.${ARCH}" \
+              "${VMTEST_CONFIGS}/DENYLIST" \
+              "${VMTEST_CONFIGS}/DENYLIST.${ARCH}" \
+              "${VMTEST_CONFIGS}/DENYLIST.${DEPLOYMENT}" \
+            2> /dev/null > "${DENYLIST_FILE}" || true
+
       - name: Run selftests
         uses: ./run-vmtest
         # https://github.com/actions/runner/issues/1483#issuecomment-1031671517

build-selftests/action.yml

@@ -1,34 +1,27 @@
 name: 'build selftests'
 description: 'Build BPF selftests'
 inputs:
-  kernel:
-    description: 'kernel version'
-    default: 'LATEST'
-  toolchain:
-    description: 'what toolchain to use'
-    default: 'gcc'
-  kbuild-output:
-    description: 'relative or absolute path to use for storing build artifacts'
+  arch:
+    description: 'arch'
+    required: true
+  kernel-root:
+    description: 'Path to the root of the kernel source tree'
     required: true
-  max-make-jobs:
-    description: 'Maximum number of jobs to use when running make (e.g argument to -j). Default: 4*nproc'
-    default: ''
   llvm-version:
     description: 'llvm version'
     required: false
-    default: '16'
-  arch:
-    description: 'arch'
-    required: true
+    default: '18'
+  toolchain:
+    description: 'gcc or llvm'
+    default: 'gcc'
 
 runs:
   using: "composite"
   steps:
     - name: build selftests
       shell: bash
-      run: |
-        kbuild_output="$(realpath ${{ inputs.kbuild-output }})"
-        export LLVM_VERSION=${{ inputs.llvm-version }}
-        ${GITHUB_ACTION_PATH}/build_selftests.sh "${{ inputs.arch }}" "${{ inputs.kernel }}" "${{ inputs.toolchain }}" "${kbuild_output}"
       env:
-        MAX_MAKE_JOBS: ${{ inputs.max-make-jobs }}
+        LLVM_VERSION: ${{ inputs.llvm-version }}
+      run: |
+        ${GITHUB_ACTION_PATH}/build_selftests.sh "${{ inputs.arch }}" "${{ inputs.toolchain }}" "${{ inputs.kernel-root }}"
+

build-selftests/build_selftests.sh

@@ -2,14 +2,15 @@
 
 set -euo pipefail
 
-THISDIR="$(cd $(dirname $0) && pwd)"
-
-source "${THISDIR}"/../helpers.sh
+source "${GITHUB_ACTION_PATH}/../helpers.sh"
 
 TARGET_ARCH="$1"
-KERNEL="$2"
-TOOLCHAIN="$3"
-export KBUILD_OUTPUT="$4"
+TOOLCHAIN="$2"
+KERNEL_ROOT="$(realpath $3)"
+
+export KBUILD_OUTPUT="${KBUILD_OUTPUT:-${KERNEL_ROOT}}"
+export VMLINUX_BTF="${VMLINUX_BTF:-${KBUILD_OUTPUT}/vmlinux}"
+export VMLINUX_H="${VMLINUX_H:-}"
 
 ARCH="$(platform_to_kernel_arch ${TARGET_ARCH})"
 CROSS_COMPILE=""
@@ -26,34 +27,21 @@ fi
 
 foldable start build_selftests "Building selftests with $TOOLCHAIN"
 
-PREPARE_SELFTESTS_SCRIPT=${THISDIR}/prepare_selftests-${KERNEL}.sh
-if [ -f "${PREPARE_SELFTESTS_SCRIPT}" ]; then
-	(cd "${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf" && ${PREPARE_SELFTESTS_SCRIPT})
-fi
-
-if [[ "${KERNEL}" = 'LATEST' ]]; then
-	VMLINUX_H=
-else
-	VMLINUX_H=${THISDIR}/vmlinux.h
-fi
-
-cd ${REPO_ROOT}/${REPO_PATH}
-
 MAKE_OPTS=$(cat <<EOF
 	ARCH=${ARCH}
 	CROSS_COMPILE=${CROSS_COMPILE}
 	CLANG=clang-${LLVM_VERSION}
 	LLC=llc-${LLVM_VERSION}
 	LLVM_STRIP=llvm-strip-${LLVM_VERSION}
-	VMLINUX_BTF=${KBUILD_OUTPUT}/vmlinux
+	VMLINUX_BTF=${VMLINUX_BTF}
 	VMLINUX_H=${VMLINUX_H}
 EOF
 )
 SELF_OPTS=$(cat <<EOF
-	-C ${REPO_ROOT}/${REPO_PATH}/tools/testing/selftests/bpf
+	-C ${KERNEL_ROOT}/tools/testing/selftests/bpf
 EOF
 )
-make ${MAKE_OPTS} headers
+make ${MAKE_OPTS} -C ${KERNEL_ROOT} headers
 make ${MAKE_OPTS} ${SELF_OPTS} clean
 make ${MAKE_OPTS} ${SELF_OPTS} -j $(kernel_build_make_jobs)
 

get-linux-source/checkout_latest_kernel.sh

@@ -6,6 +6,9 @@ source $(cd $(dirname $0) && pwd)/../helpers.sh
 
 CWD=$(pwd)
 
+# 0 means just download a snapshot
+FETCH_DEPTH=${FETCH_DEPTH:-0}
+
 echo KERNEL_ORIGIN = ${KERNEL_ORIGIN}
 echo KERNEL_BRANCH = ${KERNEL_BRANCH}
 echo REPO_PATH = ${REPO_PATH}
@@ -22,33 +25,45 @@ SNAPSHOT_URL=${KERNEL_ORIGIN}/snapshot/bpf-next-${LINUX_SHA}.tar.gz
 echo LINUX_SHA = ${LINUX_SHA}
 echo SNAPSHOT_URL = ${SNAPSHOT_URL}
 
-if [ ! -d "${REPO_PATH}" ]; then
-	echo
-	foldable start pull_kernel_srcs "Fetching kernel sources"
-
-	mkdir -p $(dirname "${REPO_PATH}")
-	cd $(dirname "${REPO_PATH}")
-	# attempt to fetch desired bpf-next repo snapshot
-	if [ -n "${SNAPSHOT_URL}" ] && \
-	   wget -U 'BPFCIBot/1.0 (bpf@vger.kernel.org)' -nv ${SNAPSHOT_URL} && \
-	   tar xf bpf-next-${LINUX_SHA}.tar.gz --totals ; then
-		mv bpf-next-${LINUX_SHA} $(basename ${REPO_PATH})
-	else
-		# but fallback to git fetch approach if that fails
-		mkdir -p $(basename ${REPO_PATH})
-		cd $(basename ${REPO_PATH})
-		git init
-		git remote add bpf-next ${KERNEL_ORIGIN}
-		# try shallow clone first
-		git fetch --depth 32 bpf-next
-		# check if desired SHA exists
-		if ! git cat-file -e ${LINUX_SHA}^{commit} ; then
-			# if not, fetch all of bpf-next; slow and painful
-			git fetch bpf-next
-		fi
-		git reset --hard ${LINUX_SHA}
-	fi
-	rm -rf ${REPO_PATH}/.git || true
-
-	foldable end pull_kernel_srcs
+if [ -d "${REPO_PATH}" ]; then
+    echo "${REPO_PATH} directory already exists, will not download kernel sources"
+    exit 0
 fi
+
+mkdir -p $(dirname "${REPO_PATH}")
+cd $(dirname "${REPO_PATH}")
+
+# attempt to fetch desired bpf-next repo snapshot
+if [[ -n "${SNAPSHOT_URL}" && "${FETCH_DEPTH}" -le 0 ]]; then
+    wget -U 'BPFCIBot/1.0 (bpf@vger.kernel.org)' -nv ${SNAPSHOT_URL} || true
+    PRESERVE_DOT_GIT=
+else
+    PRESERVE_DOT_GIT=true
+fi
+
+tarball=bpf-next-${LINUX_SHA}.tar.gz
+if [[ -f "$tarball" ]]; then
+    tar xf "$tarball" --totals
+    mv bpf-next-${LINUX_SHA} $(basename ${REPO_PATH})
+else
+    # FETCH_DEPTH can be 0 here in case snapshot download has failed
+    if [ "${FETCH_DEPTH}" -le 0 ]; then
+        FETCH_DEPTH=1
+    fi
+
+    git clone --depth ${FETCH_DEPTH} ${KERNEL_ORIGIN} ${REPO_PATH}
+
+    cd "${REPO_PATH}"
+    # check if desired SHA exists
+    if ! git cat-file -e ${LINUX_SHA}^{commit} ; then
+        # if not, fetch all of bpf-next; slow and painful
+	git fetch origin
+    fi
+    git reset --hard ${LINUX_SHA}
+    cd -
+fi
+
+if [ -z "${PRESERVE_DOT_GIT}" ]; then
+    rm -rf ${REPO_PATH}/.git || true
+fi
+

patch-kernel/patch_kernel.sh

@@ -6,7 +6,7 @@ DIFF_DIR=$1
 for ext in diff patch; do
   if ls ${DIFF_DIR}/*.${ext} 1>/dev/null 2>&1; then
     for file in ${DIFF_DIR}/*.${ext}; do
-      if patch --dry-run -p1 -s < "${file}" 2>/dev/null; then
+      if patch --dry-run -N --silent -p1 -s < "${file}" 2>/dev/null; then
         patch -s -p1 < "${file}" 2>/dev/null
         echo "Successfully applied ${file}!"
       else