testing: Import reusable workflows from kernel-patches/vmtests to tes…

…t libbpf/ci changes

Signed-off-by: Manu Bretelle <chantr4@gmail.com>

.github/scripts/get-commit-metadata.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+branch="${GITHUB_BASE_REF}"
+
+if [ "${GITHUB_EVENT_NAME}" = 'push' ]; then
+  branch="${GITHUB_REF_NAME}"
+fi
+
+echo "branch=${branch}" >> "${GITHUB_OUTPUT}"
+
+upstream="${branch//_base/}"
+commit="$(
+  git rev-parse "origin/${upstream}" &> /dev/null \
+    || (
+      git fetch --quiet --prune --no-tags --depth=1 --no-recurse-submodules origin "+refs/heads/${upstream}:refs/remotes/origin/${upstream}" && \
+      git rev-parse "origin/${upstream}"
+    )
+)"
+timestamp_utc="$(TZ=utc git show --format='%cd' --no-patch --date=iso-strict-local "${commit}")"
+
+echo "timestamp=${timestamp_utc}" >> "${GITHUB_OUTPUT}"
+echo "commit=${commit}" >> "${GITHUB_OUTPUT}"
+echo "Most recent upstream commit is ${commit}"

.github/scripts/prepare-incremental-builds.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -eu
+
+commit_id="${1}"
+
+# $1 - the SHA-1 to fetch and check out
+fetch_and_checkout() {
+  local build_base_sha
+
+  build_base_sha="${1}"
+  # If cached artifacts became stale for one reason or another, we
+  # may not have the build base SHA available. Fetch it and retry.
+  git fetch origin "${build_base_sha}" && git checkout --quiet "${build_base_sha}"
+}
+
+# $1 - value of KBUILD_OUTPUT
+clear_cache_artifacts() {
+  local output_dir
+
+  output_dir="${1}"
+  echo "Unable to find earlier upstream ref. Discarding KBUILD_OUTPUT contents..."
+  rm --recursive --force "${output_dir}"
+  mkdir "${output_dir}"
+  false
+}
+
+# $1 - value of KBUILD_OUTPUT
+# $2 - current time in ISO 8601 format
+restore_source_code_times() {
+  local build_output
+  local current_time
+  local src_time
+  local obj_time
+
+  build_output="${1}"
+  current_time="${2}"
+  src_time="$(date --iso-8601=ns --date="${current_time} - 2 minutes")"
+  obj_time="$(date --iso-8601=ns --date="${current_time} - 1 minute")"
+
+  git ls-files | xargs --max-args=10000 touch -m --no-create --date="${src_time}"
+  find "${build_output}" -type f | xargs --max-args=10000 touch -m --no-create --date="${obj_time}"
+  git checkout --quiet -
+  echo "Adjusted src and obj time stamps relative to system time"
+}
+
+mkdir --parents "${KBUILD_OUTPUT}"
+current_time="$(date --iso-8601=ns)"
+
+if [ -f "${KBUILD_OUTPUT}/.build-base-sha" ]; then
+  build_base_sha="$(cat "${KBUILD_OUTPUT}/.build-base-sha")"
+  echo "Setting up base build state for ${build_base_sha}"
+
+  (
+    git checkout --quiet "${build_base_sha}" \
+      || fetch_and_checkout "${build_base_sha}" \
+      || clear_cache_artifacts "${KBUILD_OUTPUT}"
+  ) && restore_source_code_times "${KBUILD_OUTPUT}" "${current_time}"
+else
+  echo "No previous build data found"
+fi
+
+echo -n "${commit_id}" > "${KBUILD_OUTPUT}/.build-base-sha"

.github/scripts/tar-artifact.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -eux
+
+arch="${1}"
+toolchain="${2}"
+archive_make_helpers="${3:-0}"
+
+# Convert a platform (as returned by uname -m) to the kernel
+# arch (as expected by ARCH= env).
+platform_to_kernel_arch() {
+  case $1 in
+    s390x)
+      echo "s390"
+      ;;
+    aarch64)
+      echo "arm64"
+      ;;
+    riscv64)
+      echo "riscv"
+      ;;
+    x86_64)
+      echo "x86"
+      ;;
+    *)
+      echo "$1"
+      ;;
+  esac
+}
+
+# Remove intermediate object files that we have no use for. Ideally
+# we'd just exclude them from tar below, but it does not provide
+# options to express the precise constraints.
+find selftests/ -name "*.o" -a ! -name "*.bpf.o" -print0 | \
+  xargs --null --max-args=10000 rm
+
+# Strip debug information, which is excessively large (consuming
+# bandwidth) while not actually being used (the kernel does not use
+# DWARF to symbolize stacktraces).
+"${arch}"-linux-gnu-strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux
+
+additional_file_list=()
+if [ $archive_make_helpers -ne 0 ]; then
+  # Package up a bunch of additional infrastructure to support running
+  # 'make kernelrelease' and bpf tool checks later on.
+  mapfile -t additional_file_list < <(find . -iname Makefile)
+  additional_file_list+=(
+    "scripts/" 
+    "tools/testing/selftests/bpf/" 
+    "tools/include/" 
+    "tools/bpf/bpftool/"
+  )
+fi
+
+image_name=$(make ARCH="$(platform_to_kernel_arch "${arch}")" -s image_name)
+
+# zstd is installed by default in the runner images.
+tar -cf - \
+  "${KBUILD_OUTPUT}/.config" \
+  "${KBUILD_OUTPUT}/${image_name}" \
+  "${KBUILD_OUTPUT}/include/config/auto.conf" \
+  "${KBUILD_OUTPUT}/include/generated/autoconf.h" \
+  "${KBUILD_OUTPUT}/vmlinux" \
+  "${additional_file_list[@]}" \
+  --exclude '*.cmd' \
+  --exclude '*.d' \
+  --exclude '*.h' \
+  --exclude '*.output' \
+  selftests/bpf/ | zstd -T0 -19 -o "vmlinux-${arch}-${toolchain}.tar.zst"

.github/workflows/kernel-build-test.yml

@@ -0,0 +1,94 @@
+name: Reusable Build/Test/Veristat workflow
+
+on:
+  workflow_call:
+    inputs:
+      arch:
+        required: true
+        type: string
+        description: The architecture to build against, e.g x86_64, aarch64, s390x...
+      toolchain_full:
+        required: true
+        type: string
+        description: The toolchain and for llvm, its version, e.g gcc, llvm-15
+      toolchain:
+        required: true
+        type: string
+        description: The toolchain, e.g gcc, llvm
+      runs_on:
+        required: true
+        type: string
+        description: The runners to run the test on. This is a json string representing an array of labels.
+      build_runs_on:
+        required: true
+        type: string
+        description: The runners to run the builds on. This is a json string representing an array of labels.
+      llvm-version:
+        required: true
+        type: string
+        description: The version of LLVM used to build selftest.... for llvm toolchain, this should match the one from toolchain_full, for gcc it is an arbritrary version we decide to build selftests against.
+      kernel:
+        required: true
+        type: string
+        description: The kernel to run the test against. For KPD this is always LATEST, which runs against a newly built kernel.
+      tests:
+        required: true
+        type: string
+        description: A serialized json array with the tests to be running, it must follow the json-matrix format, https://www.jitsejan.com/use-github-actions-with-json-file-as-matrix
+      run_tests:
+        required: true
+        type: boolean
+        description: Whether or not to run the test job.
+      download_sources:
+        required: true
+        type: boolean
+        description: Whether to download the linux sources into the working directory.
+        default: false
+      build_release:
+        required: true
+        type: boolean
+        description: Build selftests with -O2 optimization in addition to non-optimized build.
+        default: false
+
+jobs:
+  # Build kernel and selftest
+  build:
+    uses: ./.github/workflows/kernel-build.yml
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      toolchain: ${{ inputs.toolchain }}
+      runs_on: ${{ inputs.build_runs_on }}
+      llvm-version: ${{ inputs.llvm-version }}
+      kernel: ${{ inputs.kernel }}
+      download_sources: ${{ inputs.download_sources }}
+  build-release:
+    if: ${{ inputs.build_release }}
+    uses: ./.github/workflows/kernel-build.yml
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      toolchain: ${{ inputs.toolchain }}
+      runs_on: ${{ inputs.runs_on }}
+      llvm-version: ${{ inputs.llvm-version }}
+      kernel: ${{ inputs.kernel }}
+      download_sources: ${{ inputs.download_sources }}
+      release: true
+  test:
+    if: ${{ inputs.run_tests }}
+    uses: ./.github/workflows/kernel-test.yml
+    # Setting name to test here to avoid lengthy autogenerated names due to matrix
+    # e.g build-and-test x86_64-gcc / test (test_progs_parallel, true, 30) / test_progs_parallel on x86_64 with gcc
+    name: "test"
+    needs: [build]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.tests) }}
+    with:
+      arch: ${{ inputs.arch }}
+      toolchain_full: ${{ inputs.toolchain_full }}
+      runs_on: ${{ inputs.runs_on }}
+      kernel: ${{ inputs.kernel }}
+      test: ${{ matrix.test }}
+      continue_on_error: ${{ toJSON(matrix.continue_on_error) }}
+      timeout_minutes: ${{ matrix.timeout_minutes }}