[GLUTEN-7394][CH]Reduce the times of the calling listFiles when execu… #211

Workflow file for this run

.github/workflows/velox_backend_cache.yml at 3dceeb8

	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: Velox backend Dependency Cache

	on:
	push:
	branches:
	- 'main'

	env:
	ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true

	concurrency:
	group: ${{ github.repository }}-${{ github.workflow }}
	cancel-in-progress: false

	jobs:
	cache-native-lib-centos-7:
	runs-on: ubuntu-20.04
	container: apache/gluten:vcpkg-centos-7
	steps:
	- uses: actions/checkout@v2
	- name: Generate cache key
	run: \|
	echo ${{ hashFiles('./ep/build-velox/src/', './dev/', './cpp/', './.github/workflows/') }} > cache-key
	- name: Check existing caches
	id: check-cache
	uses: actions/cache/restore@v3
	with:
	lookup-only: true
	path: \|
	./cpp/build/releases/
	key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
	- name: Build Gluten native libraries
	if: steps.check-cache.outputs.cache-hit != 'true'
	run: \|
	df -a
	bash dev/ci-velox-buildstatic-centos-7.sh
	- name: Cache
	if: steps.check-cache.outputs.cache-hit != 'true'
	id: cache
	uses: actions/cache/save@v3
	with:
	path: \|
	./cpp/build/releases/
	key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}

	cache-native-lib-centos-8:
	runs-on: ubuntu-20.04
	container: ghcr.io/facebookincubator/velox-dev:centos8
	steps:
	- uses: actions/checkout@v2
	- name: Generate cache key
	run: \|
	echo ${{ hashFiles('./ep/build-velox/src/', './dev/', './cpp/', './.github/workflows/') }} > cache-key
	- name: Check existing caches
	id: check-cache
	uses: actions/cache/restore@v3
	with:
	lookup-only: true
	path: \|
	./cpp/build/releases/
	./cpp/build/velox/udf/examples/
	./cpp/build/velox/benchmarks/
	/root/.m2/repository/org/apache/arrow/
	key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
	- name: Setup java and maven
	if: steps.check-cache.outputs.cache-hit != 'true'
	run: \|
	sed -i -e "s\|mirrorlist=\|#mirrorlist=\|g" /etc/yum.repos.d/CentOS-* \|\| true
	sed -i -e "s\|#baseurl=http://mirror.centos.org\|baseurl=http://vault.centos.org\|g" /etc/yum.repos.d/CentOS-* \|\| true
	yum install sudo patch java-1.8.0-openjdk-devel wget -y
	bash .github/workflows/util/setup_helper.sh install_maven
	- name: Build Gluten native libraries
	if: steps.check-cache.outputs.cache-hit != 'true'
	run: \|
	df -a
	bash dev/ci-velox-buildshared-centos-8.sh
	- name: Cache
	if: steps.check-cache.outputs.cache-hit != 'true'
	id: cache
	uses: actions/cache/save@v3
	with:
	path: \|
	./cpp/build/releases/
	./cpp/build/velox/udf/examples/
	./cpp/build/velox/benchmarks/
	/root/.m2/repository/org/apache/arrow/
	key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}

	# ccache-native-lib-ubuntu-velox-ut:
	# runs-on: ubuntu-20.04
	# env:
	# CCACHE_DIR: "${{ github.workspace }}/.ccache"
	# container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
	# steps:
	# - uses: actions/checkout@v2
	# - name: Get Ccache
	# uses: actions/cache/restore@v3
	# with:
	# path: '${{ env.CCACHE_DIR }}'
	# key: ccache-ubuntu-release-default
	# - name: Ensure Cache Dirs Exists
	# working-directory: ${{ github.workspace }}
	# run: \|
	# mkdir -p '${{ env.CCACHE_DIR }}'
	# - name: Build Gluten native libraries
	# run: \|
	# rm -rf /opt/miniconda-for-velox/
	# cd ep/build-velox/src && \
	# ./get_velox.sh
	# cd ../build/velox_ep/
	# make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"

	# - name: CCache after
	# run: \|
	# ccache -vs

	# - uses: actions/cache/save@v3
	# with:
	# path: '${{ env.CCACHE_DIR }}'
	# key: ccache-ubuntu-release-default
	# ccache-native-lib-centos-velox-ut:
	# runs-on: ubuntu-20.04
	# env:
	# CCACHE_DIR: "${{ github.workspace }}/.ccache"
	# container: ghcr.io/facebookincubator/velox-dev:centos8
	# steps:
	# - uses: actions/checkout@v2
	# - name: Setup java and maven
	# run: \|
	# yum install sudo patch java-1.8.0-openjdk-devel wget -y && \
	# wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
	# tar -xvf apache-maven-3.8.8-bin.tar.gz
	# mv apache-maven-3.8.8 /usr/lib/maven
	# - name: Get Ccache
	# uses: actions/cache/restore@v3
	# with:
	# path: '${{ env.CCACHE_DIR }}'
	# key: ccache-centos-release-default
	# - name: Ensure Cache Dirs Exists
	# working-directory: ${{ github.workspace }}
	# run: \|
	# mkdir -p '${{ env.CCACHE_DIR }}'
	# - name: Build Gluten native libraries
	# run: \|
	# rm -rf /opt/miniconda-for-velox/
	# cd ep/build-velox/src && \
	# ./get_velox.sh
	# cd ../build/velox_ep/
	# source /opt/rh/gcc-toolset-9/enable
	# make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"
	#
	# - name: CCache after
	# run: \|
	# ccache -s
	#
	# - uses: actions/cache/save@v3
	# with:
	# path: '${{ env.CCACHE_DIR }}'
	# key: ccache-centos-release-default

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[GLUTEN-7394][CH]Reduce the times of the calling listFiles when execu… #211

Workflow file

[GLUTEN-7394][CH]Reduce the times of the calling listFiles when execu… #211

Jobs

Run details

Workflow file for this run