diff --git a/.github/workflows/mvn-verify-check.yml b/.github/workflows/mvn-verify-check.yml index 0466ab543a0..b9dab57d339 100644 --- a/.github/workflows/mvn-verify-check.yml +++ b/.github/workflows/mvn-verify-check.yml @@ -64,29 +64,10 @@ jobs: restore-keys: ${{ runner.os }}-maven- - name: populate-daily-cache if: steps.cache.outputs.cache-hit != 'true' + env: + SCALA_VER: '2.12' run: | - set -x - max_retry=3; delay=30; i=1 - while true; do - for pom in pom.xml - do - mvn ${{ env.COMMON_MVN_FLAGS }} --file $pom help:evaluate -pl dist \ - -Dexpression=included_buildvers \ - -DforceStdout -PnoSnapshots -q | tr -d ',' | \ - xargs -n 1 bash -c \ - 'mvn ${{ env.COMMON_MVN_FLAGS }} --file $1 -Dbuildver=$2 de.qaware.maven:go-offline-maven-plugin:resolve-dependencies' _ $pom - - # compile base versions to cache scala compiler and compiler bridge - mvn ${{ env.COMMON_MVN_FLAGS }} --file $pom \ - process-test-resources -pl sql-plugin-api -am - done && break || { - if [[ $i -le $max_retry ]]; then - echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) - else - echo "mvn command failed. Exit 1"; exit 1 - fi - } - done + . .github/workflows/mvn-verify-check/populate-daily-cache.sh - name: all shim versions id: all212ShimVersionsStep run: | @@ -167,16 +148,40 @@ jobs: fi } done - - set-scala213-versions: + + cache-dependencies-scala213: runs-on: ubuntu-latest outputs: + scala213dailyCacheKey: ${{ steps.generateCacheKey.outputs.scala213dailyCacheKey }} scala213Versions: ${{ steps.all213ShimVersionsStep.outputs.scala213Versions }} sparkJDK17Versions: ${{ steps.all213ShimVersionsStep.outputs.jdkVersions }} steps: - uses: actions/checkout@v4 # refs/pull/:prNumber/merge - - - id: all213ShimVersionsStep + - uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 17 + - name: Generate daily cache key + id: generateCacheKey + run: | + set -x + cacheKey="${{ runner.os }}-maven-scala213-${{ hashFiles('**/pom.xml') }}-${{ github.event.pull_request.base.ref }}-$(date +'%Y-%m-%d')" + echo "scala213dailyCacheKey=$cacheKey" | tee $GITHUB_ENV $GITHUB_OUTPUT + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ env.scala213dailyCacheKey }} + restore-keys: ${{ runner.os }}-maven- + - name: populate-daily-cache + if: steps.cache.outputs.cache-hit != 'true' + env: + SCALA_VER: '2.13' + run: | + . .github/workflows/mvn-verify-check/populate-daily-cache.sh + - name: all 213 shim verions + id: all213ShimVersionsStep run: | set -x SCALA_BINARY_VER=2.13 @@ -196,12 +201,13 @@ jobs: jdkVersionArrBody=${jdkVersionArrBody:1} jdkVersionJsonStr=$(printf {\"include\":[%s]} $jdkVersionArrBody) echo "jdkVersions=$jdkVersionJsonStr" >> $GITHUB_OUTPUT + package-tests-scala213: - needs: set-scala213-versions + needs: cache-dependencies-scala213 continue-on-error: ${{ matrix.isSnapshot }} strategy: - matrix: ${{ fromJSON(needs.set-scala213-versions.outputs.scala213Versions) }} + matrix: ${{ fromJSON(needs.cache-dependencies-scala213.outputs.scala213Versions) }} fail-fast: false runs-on: ubuntu-latest steps: @@ -214,6 +220,12 @@ jobs: distribution: adopt java-version: 17 + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ needs.cache-dependencies-scala213.outputs.scala213dailyCacheKey }} + - name: check runtime before tests run: | env | grep JAVA @@ -252,10 +264,10 @@ jobs: done verify-213-modules: - needs: set-scala213-versions + needs: cache-dependencies-scala213 runs-on: ubuntu-latest strategy: - matrix: ${{ fromJSON(needs.set-scala213-versions.outputs.sparkJDK17Versions) }} + matrix: ${{ fromJSON(needs.cache-dependencies-scala213.outputs.sparkJDK17Versions) }} steps: - uses: actions/checkout@v4 # refs/pull/:prNumber/merge @@ -265,6 +277,12 @@ jobs: distribution: adopt java-version: 17 + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ needs.cache-dependencies-scala213.outputs.scala213dailyCacheKey }} + - name: check runtime before tests run: | env | grep JAVA diff --git a/.github/workflows/mvn-verify-check/populate-daily-cache.sh b/.github/workflows/mvn-verify-check/populate-daily-cache.sh new file mode 100755 index 00000000000..c1c956fd7c4 --- /dev/null +++ b/.github/workflows/mvn-verify-check/populate-daily-cache.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x +max_retry=3; delay=30; i=1 +if [[ $SCALA_VER == '2.12' ]]; then + pom='pom.xml' +elif [[ $SCALA_VER == '2.13' ]]; then + pom='scala2.13/pom.xml' +fi +while true; do + { + mvn $COMMON_MVN_FLAGS --file $pom help:evaluate -pl dist \ + -Dexpression=included_buildvers \ + -DforceStdout -PnoSnapshots -q | tr -d ',' | \ + xargs -n 1 -I {} bash -c \ + "mvn $COMMON_MVN_FLAGS --file $pom -Dbuildver={} de.qaware.maven:go-offline-maven-plugin:resolve-dependencies" + + # compile base versions to cache scala compiler and compiler bridge + mvn $COMMON_MVN_FLAGS --file $pom \ + process-test-resources -pl sql-plugin-api -am + } && break || { + if [[ $i -le $max_retry ]]; then + echo "mvn command failed. Retry $i/$max_retry."; ((i++)); sleep $delay; ((delay=delay*2)) + else + echo "mvn command failed. Exit 1"; exit 1 + fi +} +done \ No newline at end of file