ci: fix macos and aarch64 builds #1922
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test and Build | |
| on: [push] | |
| env: | |
| # NOTE keep versions in sync with version in TOSCA templates | |
| TERRAFORM_VERSION: 1.1.4 # sync with tosca_plugins/artifacts.yaml | |
| HELM_VERSION: 3.7.1 # sync with configurators/helm-template.yaml | |
| GCLOUD_VERSION: 398.0.0 # sync with tosca_plugins/artifacts.yaml and Dockerfile | |
| KOMPOSE_VERSION: v1.26.1 # sync with Dockerfile | |
| KUBECTL_VERSION: v1.24.2 | |
| K3D_VERSION: v4.4.6 | |
| jobs: | |
| test: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python: [3.8, 3.9, "3.10", "3.11.0", "3.12"] | |
| env: | |
| TOX_SKIP_ENV: .+(docker|lock) | |
| UNFURL_LOGGING: info | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| check-latest: true | |
| - name: Setup Rust | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| components: clippy | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| with: | |
| terraform_version: ${{ env.TERRAFORM_VERSION }} | |
| terraform_wrapper: false | |
| - name: Setup helm | |
| uses: Azure/setup-helm@v4.0.0 | |
| with: | |
| version: ${{ env.HELM_VERSION }} | |
| - name: Setup kubectl | |
| uses: Azure/setup-kubectl@v4.0.0 | |
| with: | |
| version: ${{ env.KUBECTL_VERSION }} | |
| - name: Setup kompose | |
| run: | | |
| wget https://github.com/kubernetes/kompose/releases/download/${KOMPOSE_VERSION}/kompose-linux-amd64 | |
| chmod +x kompose-linux-amd64 | |
| sudo mv kompose-linux-amd64 /usr/local/bin/kompose | |
| - name: Setup k3d | |
| run: | | |
| wget https://github.com/rancher/k3d/releases/download/"${{ env.K3D_VERSION }}"/k3d-linux-amd64 | |
| chmod +x k3d-linux-amd64 | |
| sudo mv k3d-linux-amd64 /usr/local/bin/k3d | |
| - name: Set up the cluster | |
| run: | | |
| k3d cluster create | |
| kubectl get node | |
| k3d kubeconfig merge --all -d | |
| kubectl config view | |
| - name: Install Tox and any other packages | |
| run: pip install tox==3.28.0 | |
| - name: Run mypy and Tox | |
| # Run tox using the version of Python in `PATH` | |
| run: | | |
| git config --global user.email "unittest@onecommons.org" | |
| git config --global user.name "Test Robot" | |
| git config --global push.autoSetupRemote true | |
| git config --global init.defaultBranch main | |
| export PY_V=py3`python -c "import sys; print(sys.version_info.minor, end='')"` | |
| tox -c tosca-parser/tox.ini -v -e $PY_V | |
| tox -e $PY_V -v -- -vv -n auto --dist loadfile | |
| .tox/$PY_V/bin/mypy unfurl --install-types --non-interactive | |
| - name: build docs | |
| if: matrix.python == 3.9 | |
| run: tox -e docs -- -W --keep-going | |
| - name: upload docs | |
| if: matrix.python == 3.9 | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: DocumentationHTML | |
| path: .tox/docs/html/ | |
| publish_dockerhub: | |
| needs: test | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKERHUB_USER: "ocbuilds" | |
| # Map a step output to a job output | |
| outputs: | |
| release: ${{ steps.get_tag_name.outputs.release }} | |
| steps: | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ env.DOCKERHUB_USER }} | |
| password: ${{ secrets.DOCKER_HUB }} | |
| - name: Fetch repo | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Get the tag name or abbreviated commit digest | |
| id: get_tag_name | |
| run: | | |
| label=$(git describe --contains --always --match=v*) | |
| echo "tag=${label%^0}" >> $GITHUB_OUTPUT | |
| if [[ $label =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then | |
| echo "release=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| onecommons/unfurl | |
| tags: | | |
| type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
| type=raw,value=stable,enable=${{ steps.get_tag_name.outputs.release || 'false' }} | |
| type=raw,value=${{ steps.get_tag_name.outputs.tag }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push | |
| id: docker_build | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./ | |
| file: ./docker/Dockerfile | |
| push: true | |
| build-args: | | |
| HELM_VERSION=${{ env.HELM_VERSION }} | |
| TERRAFORM_VERSION=${{ env.TERRAFORM_VERSION }} | |
| GCLOUD_VERSION=${{ env.GCLOUD_VERSION }} | |
| KOMPOSE_VERSION=${{ env.KOMPOSE_VERSION }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| publish_dockerhub_server: | |
| needs: test | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKERHUB_USER: "ocbuilds" | |
| steps: | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ env.DOCKERHUB_USER }} | |
| password: ${{ secrets.DOCKER_HUB }} | |
| - name: Fetch repo | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Get the tag name or abbreviated commit digest | |
| id: get_tag_name | |
| run: | | |
| label=$(git describe --contains --always --match=v*) | |
| echo "tag=${label%^0}" >> $GITHUB_OUTPUT | |
| if [[ $label =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then | |
| echo "release=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Docker meta for unfurl server | |
| id: meta_server | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| onecommons/unfurl | |
| tags: | | |
| type=raw,value=latest-server,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
| type=raw,value=stable-server,enable=${{ steps.get_tag_name.outputs.release || 'false' }} | |
| type=raw,value=${{ steps.get_tag_name.outputs.tag }}-server | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push unfurl server | |
| id: docker_build_server | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./ | |
| file: ./docker/Dockerfile.server | |
| push: true | |
| tags: ${{ steps.meta_server.outputs.tags }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| publish_dockerhub_nginx: | |
| needs: [test, publish_dockerhub_server] | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKERHUB_USER: "ocbuilds" | |
| steps: | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ env.DOCKERHUB_USER }} | |
| password: ${{ secrets.DOCKER_HUB }} | |
| - name: Fetch repo | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Get the tag name or abbreviated commit digest | |
| id: get_tag_name | |
| run: | | |
| label=$(git describe --contains --always --match=v*) | |
| echo "tag=${label%^0}" >> $GITHUB_OUTPUT | |
| if [[ $label =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then | |
| echo "release=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Docker meta for unfurl server | |
| id: meta_server | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| onecommons/unfurl | |
| tags: | | |
| type=raw,value=latest-server-cached,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
| type=raw,value=stable-server-cached,enable=${{ steps.get_tag_name.outputs.release || 'false' }} | |
| type=raw,value=${{ steps.get_tag_name.outputs.tag }}-server-cached | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push unfurl server | |
| id: docker_build_server | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./ | |
| file: ./docker/Dockerfile.server-cached | |
| push: true | |
| build-args: | | |
| FROM_IMAGE=docker.io/onecommons/unfurl | |
| FROM_TAG=${{ steps.get_tag_name.outputs.tag }}-server | |
| tags: ${{ steps.meta_server.outputs.tags }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| publish_dockerhub_podman: | |
| needs: publish_dockerhub | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| env: | |
| DOCKERHUB_USER: "ocbuilds" | |
| steps: | |
| - name: Login to DockerHub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ env.DOCKERHUB_USER }} | |
| password: ${{ secrets.DOCKER_HUB }} | |
| - name: fetch repo | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: Get the tag name or abbreviated commit digest | |
| id: get_tag_name | |
| run: | | |
| label=$(git describe --contains --always --match=v*) | |
| echo "tag=${label%^0}" >> $GITHUB_OUTPUT | |
| if [[ $label =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,4}.0$ ]]; then | |
| echo "release=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Docker meta | |
| id: meta | |
| uses: docker/metadata-action@v4 | |
| with: | |
| images: | | |
| onecommons/unfurl | |
| tags: | | |
| type=raw,value=latest-podman,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} | |
| type=raw,value=stable-podman,enable=${{ steps.get_tag_name.outputs.release || 'false' }} | |
| type=raw,value=${{ steps.get_tag_name.outputs.tag }}-podman | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build and push | |
| id: docker_build | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: ./ | |
| file: ./docker/Dockerfile.podman | |
| push: true | |
| build-args: | | |
| HELM_VERSION=${{ env.HELM_VERSION }} | |
| TERRAFORM_VERSION=${{ env.TERRAFORM_VERSION }} | |
| GCLOUD_VERSION=${{ env.GCLOUD_VERSION }} | |
| KOMPOSE_VERSION=${{ env.KOMPOSE_VERSION }} | |
| UNFURL_TAG=${{ steps.get_tag_name.outputs.tag }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| build: | |
| name: > | |
| build py3.${{ matrix.python-version }} on ${{ matrix.platform || matrix.os }} | |
| (${{ matrix.alt_arch_name || matrix.arch }}) | |
| needs: | |
| - test | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu, macos] | |
| python-version: ['8'] | |
| arch: [main, alt] | |
| include: | |
| - os: ubuntu | |
| platform: linux | |
| - os: ubuntu | |
| arch: alt | |
| alt_arch_name: aarch64 | |
| - os: macos | |
| arch: alt | |
| alt_arch_name: arm64 | |
| # - os: macos | |
| # arch: main | |
| runs-on: ${{ format('{0}-latest', matrix.os) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: set up python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: set up rust | |
| if: matrix.os != 'ubuntu' | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: nightly | |
| - run: rustup target add aarch64-apple-darwin x86_64-apple-darwin | |
| if: matrix.os == 'macos' | |
| - run: rustup toolchain install stable-i686-pc-windows-msvc | |
| if: matrix.os == 'windows' | |
| - run: rustup target add i686-pc-windows-msvc | |
| if: matrix.os == 'windows' | |
| - name: install python dependencies | |
| run: pip install -U setuptools wheel twine cibuildwheel pbr | |
| - name: build sdist | |
| if: matrix.os == 'ubuntu' && matrix.python-version == '8' | |
| run: | | |
| pip install -U setuptools-rust | |
| python setup.py sdist | |
| - name: build ${{ matrix.platform || matrix.os }} binaries | |
| run: cibuildwheel --output-dir dist | |
| env: | |
| CIBW_BUILD_FRONTEND: "build" | |
| CIBW_BUILD: 'cp3${{ matrix.python-version }}-*' | |
| # rust doesn't seem to be available for musl linux on i686 | |
| # and skip rare archs to speed up builds | |
| CIBW_SKIP: '*-musllinux_i686 *ppc* *s390x' | |
| # we build for "alt_arch_name" if it exists, else 'auto | |
| CIBW_ARCHS: ${{ matrix.alt_arch_name || 'auto' }} | |
| # see https://cibuildwheel.readthedocs.io/en/stable/faq/#universal2, tests can run on cross-compiled binaries | |
| # CIBW_TEST_SKIP: '*-macosx_arm64' | |
| # CIBW_TEST_REQUIRES: pytest | |
| # CIBW_TEST_COMMAND: 'pytest {project}/tests -s' | |
| CIBW_ENVIRONMENT: 'PATH="$HOME/.cargo/bin:$PATH" MACOSX_DEPLOYMENT_TARGET=10.12' | |
| CIBW_ENVIRONMENT_WINDOWS: 'PATH="$UserProfile\.cargo\bin;$PATH"' | |
| CIBW_ARCHS_MACOS: "universal2" | |
| CIBW_MANYLINUX_I686_IMAGE: manylinux2014 | |
| CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 | |
| CIBW_MUSLLINUX_X86_64_IMAGE: musllinux_1_1 | |
| CIBW_MANYLINUX_AARCH64_IMAGE: quay.io/pypa/manylinux_2_28_aarch64:2023-11-13-f6b0c51 | |
| CIBW_MUSLLINUX_AARCH64_IMAGE: musllinux_1_1 | |
| CIBW_BEFORE_BUILD: rustup show | |
| CIBW_BEFORE_BUILD_LINUX: > | |
| curl https://sh.rustup.rs -sSf | sh -s -- --profile=minimal -y && | |
| rustup show | |
| - run: ${{ matrix.ls || 'ls -lh' }} dist/ | |
| - run: twine check dist/* | |
| - uses: actions/upload-artifact@v3 # v4 doesn't allow same filename to be reloaded, see https://github.com/actions/upload-artifact/blob/main/docs/MIGRATION.md#merging-multiple-artifacts | |
| with: | |
| name: pypi_files | |
| path: dist | |
| inspect-pypi-assets: | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: get dist artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: pypi_files | |
| path: dist | |
| - name: list dist files | |
| run: | | |
| ls -lh dist/ | |
| echo "`ls dist | wc -l` files" | |
| - name: extract and list sdist file | |
| run: | | |
| mkdir sdist-files | |
| tar -xvf dist/*.tar.gz -C sdist-files | |
| tree -a sdist-files | |
| - name: extract and list wheel file | |
| run: | | |
| ls dist/*cp3*-manylinux*x86_64.whl | head -n 1 | |
| python -m zipfile --list `ls dist/*cp3*-manylinux*x86_64.whl | head -n 1` | |
| - run: pip install twine | |
| - run: twine check dist/* | |
| publish_pypi: | |
| needs: publish_dockerhub | |
| if: needs.publish_dockerhub.outputs.release | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: fetch repo | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| fetch-depth: 0 | |
| - name: build python package | |
| run: python setup.py sdist bdist_wheel | |
| - name: release to pypi | |
| uses: pypa/gh-action-pypi-publish@v1.6.4 | |
| with: | |
| user: __token__ | |
| password: ${{ secrets.PYPI }} | |
| verbose: true | |
| docs: | |
| needs: publish_dockerhub | |
| if: github.ref == 'refs/heads/main' || github.ref == 'refs/heads/publish-docs' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Download documentation | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: DocumentationHTML | |
| path: build | |
| - name: Commit documentation changes | |
| run: | | |
| git clone ${{github.server_url}}/${{github.repository}} --branch gh-pages --single-branch gh-pages | |
| cp -r build/* gh-pages/ | |
| cd gh-pages | |
| touch .nojekyll | |
| git config --local user.email "action@github.com" | |
| git config --local user.name "GitHub Action" | |
| git add . | |
| git commit -m "Update documentation" -a || true | |
| # The above command will fail if no changes were present, so we ignore | |
| # that. | |
| - name: Push docs to gh-pages branch | |
| uses: ad-m/github-push-action@master | |
| with: | |
| branch: gh-pages | |
| directory: gh-pages | |
| github_token: ${{ secrets.GITHUB_TOKEN }} |