diff --git a/.ci/script/ensure_binary_safety.sh b/.ci/script/ensure_binary_safety.sh deleted file mode 100755 index ed88c77d..00000000 --- a/.ci/script/ensure_binary_safety.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/bash -# -# Copyright 2023 Ant Group Co., Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -set -ex -set -o pipefail - -pip install --root-user-action=ignore pwntools "pyelftools==0.29" - -BINARY=$1 - -CHECK_STATUS=$(pwn checksec $1 2>&1) - -(echo "$CHECK_STATUS" | grep -Eq "RELRO:\s*Full\sRELRO") && echo "relro enabled" || exit 1 -(echo "$CHECK_STATUS" | grep -Eq "Stack:\s*Canary\sfound") && echo "has canary" || exit 1 -(echo "$CHECK_STATUS" | grep -Eq "NX:\s*NX\senabled") && echo "nx enabled" || exit 1 -(echo "$CHECK_STATUS" | grep -Eq "PIE:\s*PIE\senabled") && echo "pie enabled" || exit 1 -(echo "$CHECK_STATUS" | grep -Eq "FORTIFY:\s*Enabled") && echo "FORTIFY enabled" || exit 1 - diff --git a/.ci/script/rename_junit_xml.sh b/.ci/script/rename_junit_xml.sh deleted file mode 100755 index c7a81429..00000000 --- a/.ci/script/rename_junit_xml.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/bash - -rm -rf test-results -mkdir -p test-results - -# renaming junit xml file to satisfy CI's requirement -for path in $(find bazel-testlogs/ -name "test.xml"); do - dir_name=$(dirname ${path}) - file_name=$(basename ${path}) - path_md5=$(echo ${path} | md5sum | cut -f1 -d ' ') - target="test-results/TEST-${path_md5}.xml" - echo "mv $path to ${target} ..." - mv ${path} ${target} -done \ No newline at end of file diff --git a/.circleci/config.yml b/.circleci/config.yml index 70190728..66f89592 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,6 +43,7 @@ workflows: engine/.* run_unittest true pkg/.* run_unittest true .bazelrc run_unittest true + .circleci/unittest-config.yml run_unittest true run_regtest: when: << pipeline.parameters.enable_regtest >> jobs: @@ -56,3 +57,4 @@ workflows: engine/.* run_regtest true pkg/.* run_regtest true .bazelrc run_regtest true + .circleci/regtest-config.yml run_regtest true diff --git a/.circleci/regtest-config.yml b/.circleci/regtest-config.yml index e88c020d..e7c3f541 100644 --- a/.circleci/regtest-config.yml +++ b/.circleci/regtest-config.yml @@ -33,36 +33,18 @@ jobs: resource_class: 2xlarge steps: - checkout - - restore_cache: - name: "Restore engine cache" - key: engine-regtest-build-{{ arch }}- - - run: - name: "Checkout devtools" - command: git clone https://github.com/secretflow/devtools.git - - run: - name: Decompress cache - command: sh devtools/decompress-build-cache.sh engine_regtest_cache - run: name: "Build binary" command: | set +e mkdir -p image cp docker/scql.Dockerfile image - docker run -d -it --name scql-dev -v $(pwd):/home/admin/dev/ -v ~/.cache/engine_regtest_cache:/root/.cache/engine_regtest_cache -w /home/admin/dev --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:latest - docker exec -it scql-dev bash -c "make && bazel build //engine/... -c opt --ui_event_filters=-info,-debug,-warning --disk_cache=~/.cache/engine_regtest_cache" + docker run -d -it --name scql-dev -v $(pwd):/home/admin/dev/ -w /home/admin/dev --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:latest + docker exec -it scql-dev bash -c "make && bazel build //engine/... -c opt --ui_event_filters=-info,-debug,-warning" docker cp scql-dev:/home/admin/dev/bazel-bin/engine/exe/scqlengine image docker cp scql-dev:/home/admin/dev/bin/scdbserver image docker cp scql-dev:/home/admin/dev/bin/scdbclient image docker stop scql-dev && docker rm scql-dev - - run: - name: Cleanup and compress cache - command: | - sh devtools/clean_disk_cache.sh ~/.cache/engine_regtest_cache - sh devtools/compress-build-cache.sh engine_regtest_cache - - save_cache: - key: engine-regtest-build-{{ arch }}-{{ .Environment.CIRCLE_BUILD_NUM }} - paths: - - ~/.cache/engine_regtest_cache.tar.gz - run: name: "Build image" command: | diff --git a/.circleci/unittest-config.yml b/.circleci/unittest-config.yml index 6bd48a95..899534cc 100644 --- a/.circleci/unittest-config.yml +++ b/.circleci/unittest-config.yml @@ -31,18 +31,11 @@ jobs: # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor docker: - - image: registry.hub.docker.com/secretflow/scql-ci:0.9 + - image: secretflow/scql-ci:latest resource_class: 2xlarge # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - - run: - name: Cancel build after set time - background: true - command: | - sleep 3600 - echo "Canceling workflow as too much time has elapsed" - curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}" - checkout - run: name: "SCDB Test" @@ -52,50 +45,32 @@ jobs: go test -timeout=30m -v ./pkg/... linux_engine_ut: docker: - - image: registry.hub.docker.com/secretflow/scql-ci:0.9 + - image: secretflow/scql-ci:latest resource_class: 2xlarge # Add steps to the job # See: https://circleci.com/docs/2.0/configuration-reference/#steps steps: - # Kill the whole ci after 1hr - - run: - name: Cancel build after set time - background: true - command: | - sleep 3600 - echo "Canceling workflow as too much time has elapsed" - curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}" - checkout - - restore_cache: - name: "Restore build engine cache" - key: engine-ut-build-{{ arch }}- - run: name: "Checkout devtools" - command: git clone https://github.com/secretflow/devtools.git + command: git clone https://github.com/secretflow/devtools.git ../devtools - run: - name: Decompress cache - command: sh devtools/decompress-build-cache.sh engine_ut_cache + name: Setup GCS + command: | + echo ${gcs_content} > ../gcs.data + # FIXME: disable BwoB due to https://github.com/bazelbuild/bazel/pull/20409 + ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json - run: name: "Engine Test" command: | set +e declare -i test_status - bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --disk_cache=~/.cache/engine_ut_cache | tee test_result.log; test_status=${PIPESTATUS[0]} - sh .ci/script/rename_junit_xml.sh + bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]} + + sh ../devtools/rename_junit_xml.sh find bazel-bin/ -executable -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz exit ${test_status} - - run: - name: Cleanup and compress cache - command: | - sh devtools/clean_disk_cache.sh ~/.cache/engine_ut_cache - sh devtools/compress-build-cache.sh engine_ut_cache - when: always - - save_cache: - key: engine-ut-build-{{ arch }}-{{ .Environment.CIRCLE_BUILD_NUM }} - paths: - - ~/.cache/engine_ut_cache.tar.gz - when: always - store_test_results: path: test-results - store_artifacts: @@ -104,80 +79,65 @@ jobs: path: test_logs.tar.gz macOS_scdb_ut: macos: - xcode: 14.2 - environment: - HOMEBREW_NO_AUTO_UPDATE: 1 + xcode: 15.1 resource_class: macos.m1.large.gen1 steps: - checkout - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - run: - name: "Install homebrew dependencies" - command: | - brew install wget go - - run: - name: "SCDB Test" - command: | - set +e - go mod tidy - go test -timeout=30m -v ./pkg/... + - run: + name: "Install homebrew dependencies" + command: | + brew install wget go + - run: + name: "SCDB Test" + command: | + set +e + go mod tidy + go test -timeout=30m -v ./pkg/... macOS_engine_ut: macos: - xcode: 14.2 - environment: - HOMEBREW_NO_AUTO_UPDATE: 1 + xcode: 15.1 resource_class: macos.m1.large.gen1 steps: + - run: + name: Cancel build after set time + background: true + command: | + sleep 3600 + echo "Canceling workflow as too much time has elapsed" + curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}" - checkout - - when: - condition: - equal: [main, << pipeline.git.branch >>] - steps: - - restore_cache: - name: "Restore engine cache" - key: engine-ut-build-{{ arch }}- - - run: - name: "Checkout devtools" - command: git clone https://github.com/secretflow/devtools.git - - run: - name: Decompress cache - command: sh devtools/decompress-build-cache.sh engine_ut_cache - - run: - name: "Install homebrew dependencies" - command: | - brew install bazel cmake ninja libomp wget go@1.19 md5sha1sum - brew link go@1.19 - (cd /opt/homebrew/Cellar/bazel/*.*.*/libexec/bin && curl -fLO https://github.com/bazelbuild/bazel/releases/download/5.4.1/bazel-5.4.1-darwin-arm64 && chmod +x bazel-5.4.1-darwin-arm64) - - run: - name: "Engine Test" - command: | - set +e - declare -i test_status - bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --disk_cache=~/.cache/engine_ut_cache | tee test_result.log; test_status=${PIPESTATUS[0]} - sh .ci/script/rename_junit_xml.sh - find bazel-bin/ -perm +111 -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz - find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz - exit ${test_status} - - run: - name: Cleanup and compress cache - command: | - sh devtools/clean_disk_cache.sh ~/.cache/engine_ut_cache - sh devtools/compress-build-cache.sh engine_ut_cache - when: always - - save_cache: - key: engine-ut-build-{{ arch }}-{{ .Environment.CIRCLE_BUILD_NUM }} - paths: - - ~/.cache/engine_ut_cache.tar.gz - when: always - - store_test_results: - path: test-results - - store_artifacts: - path: test_binary.tar.gz - - store_artifacts: - path: test_logs.tar.gz + - run: + name: "Checkout devtools" + command: git clone https://github.com/secretflow/devtools.git ../devtools + - run: + name: "Install homebrew dependencies" + command: | + brew install bazelisk cmake ninja libomp wget go@1.19 md5sha1sum + brew link go@1.19 + - run: + name: Setup GCS + command: | + echo ${gcs_content} > ../gcs.data + # FIXME: disable BwoB due to https://github.com/bazelbuild/bazel/pull/20409 + ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json + - run: + name: "Engine Test" + command: | + set +e + declare -i test_status + + bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors| tee test_result.log; test_status=${PIPESTATUS[0]} + + sh ../devtools/rename_junit_xml.sh + find bazel-bin/ -perm +111 -type f -name "*_test" -print0 | xargs -0 tar -cvzf test_binary.tar.gz + find bazel-testlogs/ -type f -name "test.log" -print0 | xargs -0 tar -cvzf test_logs.tar.gz + exit ${test_status} + - store_test_results: + path: test-results + - store_artifacts: + path: test_binary.tar.gz + - store_artifacts: + path: test_logs.tar.gz # Invoke jobs via workflows # See: https://circleci.com/docs/2.0/configuration-reference/#workflows