Enable GCS based bazel cache (#190)

secretflow · Jan 4, 2024 · 71acf4f · 71acf4f
1 parent 7563c4f
commit 71acf4f
Show file tree

Hide file tree

Showing 5 changed files with 66 additions and 168 deletions.
diff --git a/.ci/script/ensure_binary_safety.sh b/.ci/script/ensure_binary_safety.sh
diff --git a/.ci/script/rename_junit_xml.sh b/.ci/script/rename_junit_xml.sh
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -43,6 +43,7 @@ workflows:
             engine/.* run_unittest true
             pkg/.* run_unittest true
             .bazelrc run_unittest true
+            .circleci/unittest-config.yml run_unittest true
   run_regtest:
     when: << pipeline.parameters.enable_regtest >>
     jobs:
@@ -56,3 +57,4 @@ workflows:
             engine/.* run_regtest true
             pkg/.* run_regtest true
             .bazelrc run_regtest true
+            .circleci/regtest-config.yml run_regtest true
diff --git a/.circleci/regtest-config.yml b/.circleci/regtest-config.yml
@@ -33,36 +33,18 @@ jobs:
     resource_class: 2xlarge
     steps:
       - checkout
-      - restore_cache:
-          name: "Restore engine cache"
-          key: engine-regtest-build-{{ arch }}-
-      - run:
-          name: "Checkout devtools"
-          command: git clone https://github.com/secretflow/devtools.git
-      - run:
-          name: Decompress cache
-          command: sh devtools/decompress-build-cache.sh engine_regtest_cache
       - run:
           name: "Build binary"
           command: |
             set +e 
             mkdir -p image
             cp docker/scql.Dockerfile image
-            docker run -d -it --name scql-dev -v $(pwd):/home/admin/dev/ -v ~/.cache/engine_regtest_cache:/root/.cache/engine_regtest_cache -w /home/admin/dev --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:latest
-            docker exec -it scql-dev bash -c "make && bazel build //engine/... -c opt --ui_event_filters=-info,-debug,-warning --disk_cache=~/.cache/engine_regtest_cache" 
+            docker run -d -it --name scql-dev -v $(pwd):/home/admin/dev/ -w /home/admin/dev --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --cap-add=NET_ADMIN --privileged=true secretflow/release-ci:latest
+            docker exec -it scql-dev bash -c "make && bazel build //engine/... -c opt --ui_event_filters=-info,-debug,-warning" 
             docker cp scql-dev:/home/admin/dev/bazel-bin/engine/exe/scqlengine image
             docker cp scql-dev:/home/admin/dev/bin/scdbserver image
             docker cp scql-dev:/home/admin/dev/bin/scdbclient image
             docker stop scql-dev &&  docker rm scql-dev
-      - run:
-          name: Cleanup and compress cache
-          command: |
-            sh devtools/clean_disk_cache.sh ~/.cache/engine_regtest_cache
-            sh devtools/compress-build-cache.sh engine_regtest_cache
-      - save_cache:
-          key: engine-regtest-build-{{ arch }}-{{ .Environment.CIRCLE_BUILD_NUM }}
-          paths:
-            - ~/.cache/engine_regtest_cache.tar.gz
       - run:
           name: "Build image"
           command: |

diff --git a/.circleci/unittest-config.yml b/.circleci/unittest-config.yml
@@ -31,18 +31,11 @@ jobs:
     # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub.
     # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor
     docker:
-      - image: registry.hub.docker.com/secretflow/scql-ci:0.9
+      - image: secretflow/scql-ci:latest
     resource_class: 2xlarge
     # Add steps to the job
     # See: https://circleci.com/docs/2.0/configuration-reference/#steps
     steps:
-      - run:
-          name: Cancel build after set time
-          background: true
-          command: |
-            sleep 3600
-            echo "Canceling workflow as too much time has elapsed"
-            curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}"
       - checkout
       - run:
           name: "SCDB Test"
@@ -52,50 +45,32 @@ jobs:
             go test -timeout=30m -v  ./pkg/...
   linux_engine_ut:
     docker:
-      - image: registry.hub.docker.com/secretflow/scql-ci:0.9
+      - image: secretflow/scql-ci:latest
     resource_class: 2xlarge
     # Add steps to the job
     # See: https://circleci.com/docs/2.0/configuration-reference/#steps
     steps:
-      # Kill the whole ci after 1hr
-      - run:
-          name: Cancel build after set time
-          background: true
-          command: |
-            sleep 3600
-            echo "Canceling workflow as too much time has elapsed"
-            curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}"
       - checkout
-      - restore_cache:
-          name: "Restore build engine cache"
-          key: engine-ut-build-{{ arch }}-
       - run:
           name: "Checkout devtools"
-          command: git clone https://github.com/secretflow/devtools.git
+          command: git clone https://github.com/secretflow/devtools.git ../devtools
       - run:
-          name: Decompress cache
-          command: sh devtools/decompress-build-cache.sh engine_ut_cache
+          name: Setup GCS
+          command: |
+            echo ${gcs_content} > ../gcs.data
+            # FIXME: disable BwoB due to https://github.com/bazelbuild/bazel/pull/20409
+            ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json
       - run:
           name: "Engine Test"
           command: |
             set +e
             declare -i test_status
-            bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --disk_cache=~/.cache/engine_ut_cache | tee test_result.log; test_status=${PIPESTATUS[0]}
-            sh .ci/script/rename_junit_xml.sh
+            bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors | tee test_result.log; test_status=${PIPESTATUS[0]}
+            
+            sh ../devtools/rename_junit_xml.sh
             find bazel-bin/ -executable -type f -name "*_test"  -print0 | xargs -0 tar -cvzf test_binary.tar.gz
             find bazel-testlogs/ -type f -name "test.log"  -print0 | xargs -0 tar -cvzf test_logs.tar.gz
             exit ${test_status}
-      - run:
-          name: Cleanup and compress cache
-          command: |
-            sh devtools/clean_disk_cache.sh ~/.cache/engine_ut_cache
-            sh devtools/compress-build-cache.sh engine_ut_cache
-          when: always
-      - save_cache:
-          key: engine-ut-build-{{ arch }}-{{ .Environment.CIRCLE_BUILD_NUM }}
-          paths:
-            - ~/.cache/engine_ut_cache.tar.gz
-          when: always
       - store_test_results:
           path: test-results
       - store_artifacts:
@@ -104,80 +79,65 @@ jobs:
           path: test_logs.tar.gz
   macOS_scdb_ut:
     macos:
-      xcode: 14.2
-    environment:
-      HOMEBREW_NO_AUTO_UPDATE: 1
+      xcode: 15.1
     resource_class: macos.m1.large.gen1
     steps:
       - checkout
-      - when:
-          condition:
-            equal: [main, << pipeline.git.branch >>]
-          steps:
-            - run:
-                name: "Install homebrew dependencies"
-                command: |
-                  brew install wget go
-            - run:
-                name: "SCDB Test"
-                command: |
-                  set +e
-                  go mod tidy
-                  go test -timeout=30m -v ./pkg/...
+      - run:
+          name: "Install homebrew dependencies"
+          command: |
+            brew install wget go
+      - run:
+          name: "SCDB Test"
+          command: |
+            set +e
+            go mod tidy
+            go test -timeout=30m -v ./pkg/...
   macOS_engine_ut:
     macos:
-      xcode: 14.2
-    environment:
-      HOMEBREW_NO_AUTO_UPDATE: 1
+      xcode: 15.1
     resource_class: macos.m1.large.gen1
     steps:
+      - run:
+          name: Cancel build after set time
+          background: true
+          command: |
+            sleep 3600
+            echo "Canceling workflow as too much time has elapsed"
+            curl -X POST --header "Content-Type: application/json" "https://circleci.com/api/v2/workflow/${CIRCLE_WORKFLOW_ID}/cancel?circle-token=${BUILD_TIMER_TOKEN}"
       - checkout
-      - when:
-          condition:
-            equal: [main, << pipeline.git.branch >>]
-          steps:
-            - restore_cache:
-                name: "Restore engine cache"
-                key: engine-ut-build-{{ arch }}-
-            - run:
-                name: "Checkout devtools"
-                command: git clone https://github.com/secretflow/devtools.git
-            - run:
-                name: Decompress cache
-                command: sh devtools/decompress-build-cache.sh engine_ut_cache
-            - run:
-                name: "Install homebrew dependencies"
-                command: |
-                  brew install bazel cmake ninja libomp wget go@1.19 md5sha1sum
-                  brew link go@1.19
-                  (cd /opt/homebrew/Cellar/bazel/*.*.*/libexec/bin && curl -fLO https://github.com/bazelbuild/bazel/releases/download/5.4.1/bazel-5.4.1-darwin-arm64 && chmod +x bazel-5.4.1-darwin-arm64)
-            - run:
-                name: "Engine Test"
-                command: |
-                  set +e
-                  declare -i test_status
-                  bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors --disk_cache=~/.cache/engine_ut_cache | tee test_result.log; test_status=${PIPESTATUS[0]}
-                  sh .ci/script/rename_junit_xml.sh
-                  find bazel-bin/ -perm +111 -type f -name "*_test"  -print0 | xargs -0 tar -cvzf test_binary.tar.gz
-                  find bazel-testlogs/ -type f -name "test.log"  -print0 | xargs -0 tar -cvzf test_logs.tar.gz
-                  exit ${test_status}
-            - run:
-                name: Cleanup and compress cache
-                command: |
-                  sh devtools/clean_disk_cache.sh ~/.cache/engine_ut_cache
-                  sh devtools/compress-build-cache.sh engine_ut_cache
-                when: always
-            - save_cache:
-                key: engine-ut-build-{{ arch }}-{{ .Environment.CIRCLE_BUILD_NUM }}
-                paths:
-                  - ~/.cache/engine_ut_cache.tar.gz
-                when: always
-            - store_test_results:
-                path: test-results
-            - store_artifacts:
-                path: test_binary.tar.gz
-            - store_artifacts:
-                path: test_logs.tar.gz
+      - run:
+          name: "Checkout devtools"
+          command: git clone https://github.com/secretflow/devtools.git ../devtools
+      - run:
+          name: "Install homebrew dependencies"
+          command: |
+            brew install bazelisk cmake ninja libomp wget go@1.19 md5sha1sum
+            brew link go@1.19
+      - run:
+          name: Setup GCS
+          command: |
+            echo ${gcs_content} > ../gcs.data
+            # FIXME: disable BwoB due to https://github.com/bazelbuild/bazel/pull/20409
+            ../devtools/bazel_cache_setup.py --in_file=../gcs.data --out_file=../gcs.json
+      - run:
+          name: "Engine Test"
+          command: |
+            set +e
+            declare -i test_status
+
+            bazel test //engine/... -c opt --ui_event_filters=-info,-debug,-warning --test_output=errors| tee test_result.log; test_status=${PIPESTATUS[0]}
+            
+            sh ../devtools/rename_junit_xml.sh
+            find bazel-bin/ -perm +111 -type f -name "*_test"  -print0 | xargs -0 tar -cvzf test_binary.tar.gz
+            find bazel-testlogs/ -type f -name "test.log"  -print0 | xargs -0 tar -cvzf test_logs.tar.gz
+            exit ${test_status}
+      - store_test_results:
+          path: test-results
+      - store_artifacts:
+          path: test_binary.tar.gz
+      - store_artifacts:
+          path: test_logs.tar.gz
 
 # Invoke jobs via workflows
 # See: https://circleci.com/docs/2.0/configuration-reference/#workflows