diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 8c1d7ae672..0000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,829 +0,0 @@ -# Use the latest 2.1 version of CircleCI pipeline process engine. -# See: https://circleci.com/docs/2.0/configuration-reference -version: 2.1 -# In order to call CircleCI with webhook, you need to set the parameters -# and leave the default values empty. -parameters: - GHA_Actor: - type: string - default: "" - GHA_Action: - type: string - default: "" - GHA_Event: - type: string - default: "" - GHA_Meta: - type: string - default: "" - run_workflow_build: - default: true - type: boolean - - run_workflow_test: - default: true - type: boolean - - run_workflow_lint: - default: true - type: boolean - - earthly_satellite: - default: "build-vm-linux-amd64" - type: string - -executors: - # Available resource classes - https://circleci.com/product/features/resource-classes/ - linux-amd64: - machine: - image: default - resource_class: large - linux-arm64: - machine: - image: default - resource_class: arm.large - darwin-arm64: - macos: - xcode: "14.2.0" - resource_class: macos.m1.medium.gen1 - windows-amd64: - machine: - image: "windows-server-2022-gui:current" - resource_class: windows.large - docker-linux: - docker: - - image: cimg/base:2022.09 - auth: - username: $DOCKERHUB_USERNAME - password: $DOCKERHUB_PASSWORD - resource_class: large - -commands: - install_go: - description: Install Golang and dependencies. - parameters: - executor: - type: enum - enum: - [ - "linux-amd64", - "linux-arm64", - "darwin-arm64", - "windows-amd64", - ] - steps: - - run: - name: Set GOVER - command: | - go_spec=$(grep 'go [[:digit:]].[[:digit:]]*' go.work | cut -d' ' -f2) - version=$(curl --silent --show-error --location --fail "https://go.dev/dl/?mode=json&include=all" | \ - jq --arg v "$go_spec" --raw-output '[.[] | select(.stable) | select(.version | startswith("go"+$v)) | .version | ltrimstr("go")] | sort_by(split(".") | map(tonumber)) | reverse | .[0]') - echo "Latest Go version for $go_spec is $version" - echo "export GOVER=$version" >> $BASH_ENV - - - when: - condition: - equal: ["windows-amd64", << parameters.executor >>] - steps: - - run: - name: Install Go - shell: bash.exe - command: | - rm -rf /c/Program\ Files/Go - curl --silent --show-error --location --fail https://go.dev/dl/go$GOVER.<< parameters.executor >>.zip | tar --extract --gzip --file=- --directory=/c/Program\ Files - - - when: - condition: - or: - - equal: ["darwin-arm64", << parameters.executor >>] - - equal: ["linux-amd64", << parameters.executor >>] - - equal: ["linux-arm64", << parameters.executor >>] - steps: - - run: - name: Install Go - command: | - sudo rm -fr /usr/local/go /usr/local/bin/go - curl --silent --show-error --location --fail https://go.dev/dl/go$GOVER.<< parameters.executor >>.tar.gz | sudo tar --extract --gzip --file=- --directory=/usr/local - sudo ln -s /usr/local/go/bin/go /usr/local/bin/go - - - run: - name: Set GOCACHE - command: echo "export GOCACHE=$HOME/.cache/go-build" >> $BASH_ENV - - - run: - name: Set GOPATH - command: echo "export GOPATH=$HOME/go" >> $BASH_ENV - - - restore_cache: - name: Restoring Go cache - key: go-mod-{{ arch }}-{{ checksum "go.sum" }} - - - run: - name: Install packages - command: go mod tidy - - - save_cache: - name: Saving Go cache - key: go-mod-{{ arch }}-{{ checksum "go.sum" }} - paths: - - ~/.cache/go-build - - ~/go/pkg/mod - - # Installs asdf, a version manager for golang, python, nodejs, etc. Installs from .tool-versions - install_asdf: - parameters: - cache_version: - type: string - default: v1-1 - asdf_version: - type: string - default: v0.14.0 - description: Install asdf and dependencies. - steps: - - restore_cache: - keys: - - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}-{{ checksum "poetry.lock" }} - - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }} - - asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }} - - asdf-<< parameters.cache_version >>- - - run: - name: Install asdf - command: | - git config advice.detachedHead false - if [ ! -d $HOME/.asdf ]; then - git clone https://github.com/asdf-vm/asdf.git $HOME/.asdf --branch << parameters.asdf_version >> - cd $HOME/.asdf - else - cd $HOME/.asdf - git fetch origin - fi - git checkout master - git pull --all - git checkout tags/<< parameters.asdf_version >> - cd - - - save_cache: - key: asdf-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }}-{{ checksum "poetry.lock" }} - paths: - - $HOME/.asdf - - run: - name: Set PATH - command: | - echo 'export PATH=$HOME/.asdf/bin:$HOME/.asdf/shims:"${PATH}"' >> "${BASH_ENV}" - source "${BASH_ENV}" - - # Installs and caches python and dependencies. Installs from .tool-versions in the local directory - # which is a file that asdf uses to install the correct version of python. - install_python: - description: Install Python and dependencies. - parameters: - cache_version: - type: string - default: v1-1 - steps: - - install_asdf: - cache_version: << parameters.cache_version >> - - restore_cache: - keys: - - python-deps-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }} - - python-deps-<< parameters.cache_version >>-{{ checksum ".tool-versions" }} - - python-deps-<< parameters.cache_version >>- - - run: - name: Install toolchain from .tool-versions - command: | - asdf plugin-add python - asdf install python $(awk '/^python/ {print $2}' .tool-versions) - pip3 install --upgrade pip - pip3 install poetry # Install poetry in the newly created virtual environment - python -m poetry install - - run: - name: Install Python API client pre-requisites - command: | - CODEGEN_BASE_URL="https://repo1.maven.org/maven2/io/swagger/codegen/v3" - wget ${CODEGEN_BASE_URL}/swagger-codegen-cli/3.0.52/swagger-codegen-cli-3.0.52.jar -O ${HOME}/bin/swagger-codegen-cli.jar - chmod +x ${HOME}/bin/swagger-codegen-cli.jar - ${HOME}/bin/swagger-codegen-cli.jar version - - save_cache: - key: python-deps-<< parameters.cache_version >>-{{ checksum ".tool-versions" }}-{{ checksum ".circleci/config.yml" }} - paths: - - $HOME/.asdf - - $HOME/.cache/pip - - $HOME/.cache/pypoetry - - $HOME/.local - - run: - name: Set PATH - # Sources PATH after installing asdf and python, from $BASH_ENV, which CircleCI uses to set the PATH - command: echo 'export PATH="${PATH}":$HOME/.asdf/bin:$HOME/.asdf/shims:$HOME/.brew/bin' >> "${BASH_ENV}" - - # When running the makefile, we need to ensure that we are using the correct shell - # so that the right shims are used. We cannot use global versions of tools because - # the makefile is not aware of the asdf environment. - run_make_under_specific_shell: - parameters: - directory_to_change_to: - type: string - makefile_directory: - type: string - make_command: - type: string - steps: - - run: - name: Run make for << parameters.make_command >> in << parameters.directory_to_change_to >> against make file in << parameters.makefile_directory >> - command: | - source $BASH_ENV - pushd << parameters.directory_to_change_to >> - python -m poetry install - source $(python -m poetry env info --path)/bin/activate # Doing this so it doesn't create a new shell - pip install poetry # Installing inside the virtual env - poetry install # Installing all the dependencies - make << parameters.make_command >> -C << parameters.makefile_directory >> - deactivate - # Return to the original directory - popd - install_earthly: - description: Install Earthly and dependencies. - steps: - - install_asdf: - cache_version: v1-1 - - run: - name: Install Earthly - command: | - asdf plugin-add earthly - asdf install earthly $(awk '/^earthly/ {print $2}' .tool-versions) - earthly --version - - select_earthly_satellite: - description: Select Earthly satellite - steps: - - run: - name: Select Earthly satellite - command: | - earthly account login - earthly sat select << pipeline.parameters.earthly_satellite >> - - install_ipfs: - parameters: - cache_version: - type: string - default: v1-1 - ipfs_version: - type: string - default: v0.18.0 - description: Install IPFS for linux-amd64 - steps: - - restore_cache: - keys: - - ipfs-<< parameters.cache_version >>-<< parameters.ipfs_version >>-{{ checksum ".circleci/config.yml" }} - - ipfs-<< parameters.cache_version >>-<< parameters.ipfs_version >> - - ipfs-<< parameters.cache_version >> - - run: - name: Install IPFS - command: | - if [ ! -f /usr/local/bin/ipfs ]; then - wget https://dist.ipfs.tech/go-ipfs/<< parameters.ipfs_version >>/go-ipfs_<< parameters.ipfs_version >>_linux-amd64.tar.gz -O ipfs.tar.gz - tar -xzf ipfs.tar.gz - cd go-ipfs - sudo bash install.sh - ipfs --version - else - echo "IPFS is already installed, skipping download." - fi - - save_cache: - key: ipfs-<< parameters.cache_version >>-<< parameters.ipfs_version >>-{{ checksum ".circleci/config.yml" }} - paths: - - /usr/local/bin/ipfs - - run: - name: Initialize IPFS - command: | - ipfs init - ipfs config Addresses.API /ip4/127.0.0.1/tcp/5001 - ipfs config Addresses.Gateway /ip4/0.0.0.0/tcp/8080 - - run: - name: Start IPFS Daemon - command: | - ipfs daemon --offline - background: true -jobs: - build_cli: - executor: linux-amd64 - steps: - - checkout - - install_asdf - - install_earthly - - attach_workspace: - at: . - - install_go: - executor: linux-amd64 - - run: - name: Build tarball - command: | - source "${BASH_ENV}" - echo "$PRIVATE_PEM_B64" | base64 --decode > /tmp/private.pem - echo "$PUBLIC_PEM_B64" | base64 --decode > /tmp/public.pem - export PRIVATE_KEY_PASSPHRASE="$(echo $PRIVATE_KEY_PASSPHRASE_B64 | base64 --decode)" - # Prevent rebuilding web ui, we should have already attached it - find webui -exec touch -c '{}' + - GOOS=linux GOARCH=amd64 make build-bacalhau-tgz - GOOS=linux GOARCH=arm64 make build-bacalhau-tgz - GOOS=darwin GOARCH=amd64 make build-bacalhau-tgz - GOOS=darwin GOARCH=arm64 make build-bacalhau-tgz - GOOS=linux GOARCH=armv6 make build-bacalhau-tgz - GOOS=linux GOARCH=armv7 make build-bacalhau-tgz - GOOS=windows GOARCH=amd64 make build-bacalhau-tgz - - - persist_to_workspace: - root: dist/ - paths: - - "*.tar.gz" - - "*.sha256" - - - store_artifacts: - path: dist/ - - test_cli_bash: - executor: linux-amd64 - steps: - - checkout - - attach_workspace: - at: . - - - run: - command: | - mkdir -p bin/linux/amd64 - pushd bin/linux/amd64 - tar -xvf ../../../bacalhau_*_linux_amd64.tar.gz - popd - - - run: cd test && bin/bashtub *.sh - - build_webui: - executor: docker-linux - steps: - - checkout - - setup_remote_docker - - install_earthly - - select_earthly_satellite - - run: - name: Build webui - command: | - cd webui - earthly --push +all - - persist_to_workspace: - root: . - paths: - - "webui/build/*" - - "webui/node_modules/*" - - build_canary: - environment: - GOPROXY: https://proxy.golang.org - TARGET_COMMIT: << pipeline.git.revision >> - working_directory: ~/repo - executor: linux-amd64 - steps: - - checkout - - - install_go: - executor: linux-amd64 - - - run: - command: | - echo "CANARIES HAVE BEEN MOVED TO https://github.com/bacalhau-project/canaries" - - # - run: - # name: Install Executor Plugins - # command: make install-plugins - - # - run: - # name: Set canary dependency - # working_directory: ops/aws/canary/lambda - # command: make update - - # - run: - # name: Build - # working_directory: ops/aws/canary/lambda - # command: make build -j - - # - run: - # name: Run tests - # working_directory: ops/aws/canary/lambda - # command: | - # export GOBIN=${HOME}/bin - # export PATH=$GOBIN:$PATH - # go install gotest.tools/gotestsum@v1.8.2 - # make test - - # - store_test_results: - # path: ops/aws/canary/lambda/tests.xml - - build_docker: - executor: linux-amd64 - steps: - - checkout - - attach_workspace: - at: dist/ - - run: - command: | - mkdir -p bin/linux/amd64 && tar -xvf dist/bacalhau_*_linux_amd64.tar.gz -C bin/linux/amd64 - mkdir -p bin/linux/arm64 && tar -xvf dist/bacalhau_*_linux_arm64.tar.gz -C bin/linux/arm64 - - run: - command: | - docker context create buildx-build - docker buildx create --use buildx-build - make build-bacalhau-image - - when: - condition: - matches: - value: << pipeline.git.tag >> - pattern: "^v([0-9]+).([0-9]+).([0-9]+).*$" - steps: - - run: - name: Login to Github Container Registry - command: echo $GHCR_PAT | docker login ghcr.io -u bacalhau-infra-bot --password-stdin - - run: make push-bacalhau-image - - build_python_packages: - executor: linux-amd64 - environment: - TARGET_COMMIT: << pipeline.git.revision >> - # PYPI_VERSION: 0.3.24.dev8 # use this to set a custom version identifier (https://peps.python.org/pep-0440/) - working_directory: ~/repo - steps: - - checkout - - install_python - - install_earthly - - select_earthly_satellite - - attach_workspace: - at: /tmp/swagger_spec/ - - run: - name: Copy swagger.json from workspace - command: | - cp /tmp/swagger_spec/webui/public/swagger/swagger.json ./docs/swagger.json - - run: - name: Build Python API client - command: | - source $BASH_ENV - make build-python-apiclient - - persist_to_workspace: - root: clients/python/dist - paths: - - bacalhau_apiclient-*.tar.gz - - bacalhau_apiclient-*.whl - - run: - name: Build Python SDK - command: make build-python-sdk - - persist_to_workspace: - root: python/dist - paths: - - bacalhau_sdk-*.tar.gz - - bacalhau_sdk-*.whl - - run: - name: Build the airflow integration - command: make build-bacalhau-airflow - - persist_to_workspace: - root: integration/airflow/dist - paths: - - bacalhau_airflow-*.tar.gz - - bacalhau_airflow-*.whl - - run: - name: Disabled Flyte - command: echo "FLYTE BUILDING DISABLED" - test_cli: - environment: - GOPROXY: https://proxy.golang.org - GOOS: << parameters.target_os >> - GOARCH: << parameters.target_arch >> - GCS_TEST_RESULTS_BUCKET: bacalhau-global-storage/test-results - GOMAXPROCS: "2" # Avoid using all the allocated CPU, so tests don't get rate limited - working_directory: ~/repo - executor: << parameters.executor >> - parameters: - executor: - type: enum - enum: - [ - "linux-amd64", - "linux-arm64", - "darwin-arm64", - "windows-amd64", - ] - target_arch: - type: enum - enum: ["amd64", "arm64", "armv6", "armv7"] - target_os: - type: enum - enum: ["linux", "darwin", "windows"] - build_tags: - type: string - default: "" - steps: - - checkout - - - attach_workspace: - at: . - - - when: - condition: - equal: ["windows", << parameters.target_os >>] - steps: - - run: - name: Install GNU Make - command: | - choco install -y make - choco install -y jq - shell: powershell.exe - - - install_go: - executor: << parameters.executor >> - - - run: - name: "Setup BACALHAU_ENVIRONMENT environment variable" - command: echo 'export BACALHAU_ENVIRONMENT=test' >> "$BASH_ENV" - - - when: - condition: - equal: ["linux-amd64", << parameters.executor >>] - steps: - - when: - condition: - equal: ["integration", << parameters.build_tags >>] - steps: - - install_ipfs - - run: - name: Set IPFS Connect Env - command: echo 'export BACALHAU_NODE_IPFS_CONNECT=/ip4/127.0.0.1/tcp/5001' >> "$BASH_ENV" - - - run: - name: Test Go - environment: - LOG_LEVEL: debug - TEST_BUILD_TAGS: << parameters.build_tags >> - TEST_PARALLEL_PACKAGES: 4 # This is set to 4 as large instances have at least 8 CPUs, and we want to leave some CPU for the Docker instances - command: | - export GOBIN=${HOME}/bin - export PATH=$GOBIN:$PATH - go install gotest.tools/gotestsum@v1.8.2 - make test-and-report - no_output_timeout: 20m - - - store_test_results: - path: . - - persist_to_workspace: - root: coverage/ - paths: - - "*.coverage" - - test_python_sdk: - executor: linux-amd64 - steps: - - checkout - - install_earthly - - select_earthly_satellite - - run: - working_directory: python - command: | - earthly --push +run-unittest - - test_python_airflow: - executor: linux-amd64 - steps: - - checkout - - install_earthly - - select_earthly_satellite - - run: - working_directory: integration/airflow - command: earthly --push +run-tests - - coverage: - executor: linux-amd64 - environment: - GOPROXY: https://proxy.golang.org - steps: - - checkout - - - attach_workspace: - at: coverage/ - - - run: - name: Install gocovmerge - command: go install github.com/wadey/gocovmerge@latest - - - run: - name: Build coverage report - command: make coverage-report - - - store_artifacts: - path: coverage/coverage.html - - release_cli: - executor: linux-amd64 - steps: - - checkout - - attach_workspace: - at: dist/ - - run: - name: Install gh - command: | - wget https://github.com/cli/cli/releases/download/v2.14.7/gh_2.14.7_linux_amd64.deb - sudo dpkg -i gh_2.14.7_linux_amd64.deb - echo "$BACALHAU_RELEASE_TOKEN" | gh auth login --with-token - - run: - name: Uploading to Release - << pipeline.git.tag >> - command: | - TAG="<< pipeline.git.tag >>" - echo "TAG = ${TAG}" - find dist/ - gh release upload $TAG dist/bacalhau_$TAG_* - - release_python: - executor: linux-amd64 - working_directory: ~/repo - steps: - - checkout - - install_python - - install_earthly - - attach_workspace: - at: /tmp/py_dist - - run: - name: Copy dist files - command: | - mkdir -p clients/python/dist - cp /tmp/py_dist/bacalhau_apiclient* clients/python/dist - mkdir -p python/dist - cp /tmp/py_dist/bacalhau_sdk* python/dist - mkdir -p integration/airflow/dist - cp /tmp/py_dist/bacalhau_airflow* integration/airflow/dist - mkdir -p integration/airflow/dist - - run: - name: Release python apiclient - command: | - python -m poetry run make release-python-apiclient - - run: - name: Release python sdk - command: | - export RELEASE_PYTHON_PACKAGES=1 - make release-python-sdk - - run: - name: Release python Airflow integration - command: | - export RELEASE_PYTHON_PACKAGES=1 - make dist - - run: - name: Release python Flyte integration - command: | - echo "FLYTE BUILDING DISABLED" - - build_swagger: - executor: linux-amd64 - environment: - GOPROXY: https://proxy.golang.org - TARGET_COMMIT: << pipeline.git.revision >> - working_directory: ~/repo - steps: - - checkout - - add_ssh_keys: - fingerprints: - - "b2:46:a2:7c:94:1f:84:be:99:70:1e:44:50:1e:33:2b" - - install_go: - executor: linux-amd64 - - run: - name: Build Swagger - command: | - export GOBIN=${HOME}/bin - export PATH=$GOBIN:$PATH - go install golang.org/x/tools/cmd/stringer - go install go.uber.org/mock/gomock - go install github.com/golang/mock/mockgen - go install github.com/swaggo/swag/cmd/swag@latest - make generate - - - persist_to_workspace: - root: . - paths: - - pkg/swagger/docs.go - - webui/public/swagger/swagger.json - - # - when: # Only if the current branch is main, we push the swagger spec to the repo (via auto PR) - # condition: - # equal: ["main", << pipeline.git.branch >>] - # steps: - # - run: - # name: Install gh - # command: | - # wget https://github.com/cli/cli/releases/download/v2.14.7/gh_2.14.7_linux_amd64.deb - # sudo dpkg -i gh_2.14.7_linux_amd64.deb - # echo "$GITHUB_PR_PAT" | gh auth login --with-token - # rm gh_2.14.7_linux_amd64.deb - # - run: - # name: Install human-id generator - # command: | - # pip3 install human-id - # - run: - # name: Git commit - # command: | - # git config user.email "$(git log --format='format:%ae' -n1)" - # git config user.name "$(git log --format='format:%an' -n1)" - # # Note: we delete this branch after the PR is merged - # GH_BRANCH_NAME=(ci-build-swagger-docs_$(humanid-gen --words 3 | grep -oE "^([^-]*-){1}[^-]*")-$(((RANDOM % $((100 - 1))) + 1))) - # git checkout -b $GH_BRANCH_NAME - # if test -n "$(git ls-files --modified | grep -e '^docs/')"; then - # git add --verbose -- ./docs - # COMMIT_MSG="Build swagger reference - this is an automatic commit" - # git commit -m "[circleci] $COMMIT_MSG [skip ci]" - # git push --set-upstream origin $GH_BRANCH_NAME - # # Note: if you close the PR below manually, you should delete the `ci-build-swagger-docs_*` branch as well - # PR_URL=$(gh pr create --fill --head $(git rev-parse --abbrev-ref HEAD) --base main --label schema --repo bacalhau-project/bacalhau) - # echo "Pull request: $PR_URL" - # sleep 3 - # gh pr merge --auto --delete-branch -r $PR_URL - # fi - -# Invoke jobs via workflows -# See: https://circleci.com/docs/2.0/configuration-reference/#workflows -workflows: - check_canary: - when: << pipeline.parameters.GHA_Action >> - jobs: - - build_canary: - name: Check canary build - filters: - tags: - ignore: /.*/ - test_workflow: - when: << pipeline.parameters.GHA_Action >> - jobs: - - build_webui - - test_cli: - name: test-<< matrix.target_os >>-<< matrix.target_arch >>-<< matrix.build_tags >> - executor: << matrix.target_os >>-<< matrix.target_arch >> - requires: - - build_webui - matrix: - parameters: - target_os: ["linux", "darwin", "windows"] - target_arch: ["amd64", "arm64"] - build_tags: ["unit", "integration"] - exclude: - - target_os: windows - target_arch: arm64 - build_tags: unit - - target_os: windows - target_arch: arm64 - build_tags: integration - - target_os: darwin - target_arch: amd64 - build_tags: unit - - target_os: darwin - target_arch: amd64 - build_tags: integration - - test_python_sdk - - test_python_airflow - - coverage: - name: Build coverage report - requires: - - test_cli - - build_python: - when: << pipeline.parameters.GHA_Action >> - jobs: - - build_swagger: - name: build-swagger-spec - filters: &build_on_branch_and_tag_push - tags: - # only run on tags that look like release tags e.g. v0.1.2 or - # v0.1.3alpha19 (actually v0.1.3anything...) - only: /^v([0-9]+).([0-9]+).([0-9]+).*$/ - - build_python_packages: - name: build-python-packages - requires: [build-swagger-spec] - filters: *build_on_branch_and_tag_push - - release_python: - name: release-python-packages - requires: - - build-swagger-spec - - build-python-packages - # This job will only run on tags (specifically starting with 'v.') and - # will not run on branches - filters: &build_on_release_only - branches: - ignore: /.*/ # don't run on any branches - only tags - <<: *build_on_branch_and_tag_push - - build_workflow: - when: << pipeline.parameters.GHA_Action >> - jobs: - - build_webui: - context: - - CREDENTIALS - filters: *build_on_branch_and_tag_push - - build_cli: - requires: [build_webui] - filters: *build_on_branch_and_tag_push - - test_cli_bash: - requires: [build_cli] - filters: *build_on_branch_and_tag_push - - build_docker: - requires: [build_cli] - filters: *build_on_branch_and_tag_push - - release_cli: - requires: [build_cli] - filters: *build_on_release_only diff --git a/.circleci/trigger_pipeline.sh b/.circleci/trigger_pipeline.sh deleted file mode 100755 index 172d1e4d53..0000000000 --- a/.circleci/trigger_pipeline.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -xeuo pipefail -IFS=$'\n\t' - -if [[ -z "${BRANCH}" ]]; then - TARGET="\"branch\": \"main\"" - elif [[ "${BRANCH}" =~ "refs/tags" ]]; then - TAG=$(echo "${BRANCH}" | sed 's:refs/tags/::') - TARGET="\"tag\": \"${TAG}\"" -else - TARGET="\"branch\": \"${BRANCH}\"" -fi - -curl --fail -X POST --header "Content-Type: application/json" --header "Circle-Token: ${CIRCLE_TOKEN}" -d "{ - \"parameters\": { - \"GHA_Action\": \"trigger_pipeline\" - }, - ${TARGET} -}" https://circleci.com/api/v2/project/gh/bacalhau-project/bacalhau/pipeline diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1ab32ba32b..4a2fda53c2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,18 +1,25 @@ version: 2 updates: -- package-ecosystem: "gomod" - directory: "/" - schedule: - interval: daily - labels: - - "lang/go" - - "th/dependencies" - open-pull-requests-limit: 1 -- package-ecosystem: "pip" - directory: "/integration/" - schedule: - interval: daily - labels: - - "lang/python" - - "th/dependencies" - open-pull-requests-limit: 1 + - package-ecosystem: "gomod" + directory: "/" + schedule: + interval: "daily" + labels: + - "lang/go" + - "dependabot/version-updates" + open-pull-requests-limit: 1 + - package-ecosystem: "pip" + directory: "/integration/" + schedule: + interval: "daily" + labels: + - "lang/python" + - "dependabot/version-updates" + open-pull-requests-limit: 1 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + labels: + - "lang/github-actions" + - "dependabot/version-updates" diff --git a/.github/workflows/circleci.yml b/.github/workflows/circleci.yml deleted file mode 100644 index ebc945d482..0000000000 --- a/.github/workflows/circleci.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: CircleCI Trigger Action -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - push: - branches: - - main - tags: - - v** - workflow_run: - workflows: ["Dependabot", "Trigger on Comment"] - types: - - completed - repository_dispatch: - types: [ok-to-test] - -permissions: - contents: write - pull-requests: write - -jobs: - trigger-circle-ci: - if: | - !contains(github.event.pull_request.labels.*.name, 'do-not-merge') && - !contains(github.event.pull_request.labels.*.name, 'wip') - runs-on: ubuntu-latest - name: Pull Request Triggered CircleCI - steps: - # For Debugging Below - - name: Dump GitHub context - env: - GITHUB_CONTEXT: ${{ toJson(github) }} - run: echo "$GITHUB_CONTEXT" - - name: Dump job context - env: - JOB_CONTEXT: ${{ toJson(job) }} - run: echo "$JOB_CONTEXT" - - name: Dump steps context - env: - STEPS_CONTEXT: ${{ toJson(steps) }} - run: echo "$STEPS_CONTEXT" - - name: Dump runner context - env: - RUNNER_CONTEXT: ${{ toJson(runner) }} - run: echo "$RUNNER_CONTEXT" - - uses: actions/checkout@v3 - - name: Trigger CircleCI - id: trigger-circle-ci - run: | - if [[ -z $CIRCLE_TOKEN ]]; then - echo "CIRCLE_TOKEN is not set. Exiting." - exit 1 - fi - ./.circleci/trigger_pipeline.sh - env: - CIRCLE_TOKEN: ${{ secrets.CIRCLE_CI_TOKEN }} - BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.event.ref }} diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml deleted file mode 100644 index 5df40cad24..0000000000 --- a/.github/workflows/dependabot-auto-merge.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Dependabot Auto Merge - -on: - pull_request: - types: [opened, synchronize, reopened] - branches: - - main - paths-ignore: - - "**.md" - repository_dispatch: - types: [ok-to-test] - -jobs: - trigger-circleci: - runs-on: ubuntu-latest - if: github.actor == 'dependabot[bot]' - steps: - - name: Trigger CircleCI Pipeline - run: .circleci/trigger_pipeline.sh - env: - BRANCH: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.ref || github.event.ref }} - CIRCLE_TOKEN: ${{ secrets.CIRCLE_CI_TOKEN }} - - auto-merge: - runs-on: ubuntu-latest - needs: trigger-circleci - steps: - - name: Auto merge PR - uses: actions/github-script@v3 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const mergePayload = { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - merge_method: 'squash' - }; - github.pulls.merge(mergePayload); diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml deleted file mode 100644 index c3a8f6f4dc..0000000000 --- a/.github/workflows/dependabot.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Dependabot -on: - pull_request: - types: [opened, synchronize, reopened, labeled, unlabeled, ready_for_review] - branches: - - dependabot/** - repository_dispatch: - types: [ok-to-test] - -permissions: - contents: write - pull-requests: write - -jobs: - update-go-package-metadata: - runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' }} - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.ref }} - - - name: Detect Go version - id: runtimes - run: echo "::set-output name=golang-version::$(grep 'go [[:digit:]].[[:digit:]]*' go.work | cut -d' ' -f2)" - - - name: Install Golang - uses: actions/setup-go@v4 - with: - go-version: ${{ steps.runtimes.outputs.golang-version }} - - - name: Update go package metadata - run: go work sync; go mod tidy - - - name: Commit any changes - continue-on-error: true - run: | - git config user.name "$(git show -s --format='%aN' HEAD)" - git config user.email "$(git show -s --format='%aE' HEAD)" - git commit -a -m "Update go metadata" && git push origin HEAD:${{github.event.pull_request.head.ref}} - env: - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} - - enable-auto-merge: - runs-on: ubuntu-latest - if: ${{ github.actor == 'dependabot[bot]' }} - steps: - - name: Fetch Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@v1 - with: - github-token: "${{ secrets.GITHUB_TOKEN }}" - - - name: Enable auto-merge for Dependabot PRs - run: gh pr merge --auto --squash "$PR_URL" - env: - PR_URL: ${{github.event.pull_request.html_url}} - GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}} diff --git a/.github/workflows/testcontainers-integration-tests.yml b/.github/workflows/testcontainers-integration-tests.yml index f349a817bb..5a28cdff1a 100644 --- a/.github/workflows/testcontainers-integration-tests.yml +++ b/.github/workflows/testcontainers-integration-tests.yml @@ -13,23 +13,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - # Detect Go version from test_integration/go.mod - - name: Detect Go version from test_integration/go.mod - id: detect-test-go-version - shell: bash - run: | - set -euo pipefail - go_version=$(grep '^go ' test_integration/go.mod | awk '{print $2}') - echo "Go version detected for integration tests: $go_version" - echo "GOLANG_TEST_VERSION=$go_version" >> $GITHUB_ENV - - - name: Set up Go + - name: Install Golang uses: actions/setup-go@v5 with: - go-version: ${{ env.GOLANG_TEST_VERSION }} - cache: true + go-version-file: go.work - name: Run Go tests in integration test directory shell: bash diff --git a/README.md b/README.md index 3844ecaf65..74e325e377 100644 --- a/README.md +++ b/README.md @@ -134,8 +134,6 @@ The build parses the OpenAPI annotations as well as the markdown files in `docs/ * `docs/swagger.json` * `docs/swagger.yaml` -Note that the CI pipeline will open and automatically merge a pull request titled `[circleci] Build swagger reference - this is an automatic commit ...` containing *only* the updated spec files. - ### Python Libraries We ship two Python Bacalhau libraries: