build-test-distribute.yml

name: Build Test Distribute

# Controls when the action will run.
on:
  # Triggers the workflow on push or pull request events but only for the master branch
  push:
    branches: [ master ]
  pull_request:
    branches: [ master ]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
  schedule:
    - cron: '0 17 * * *'  # Run every day at 17:00 UTC
    #- cron: '0 6 * * *'  # Run every day at 6 AM UTC

jobs:
  # NOTE: all output values are strings
  # use the ${{ needs.config.outputs.bool-param == 'true' }} construction to check boolean values
  config:
    uses: ./.github/workflows/config.yml

  debug-config:
    needs: config
    runs-on: ubuntu-latest
    steps:
      - name: Print config
        run: |
          echo << EOF
          ${{ toJSON( needs.config.outputs ) }}
          EOF

  prepare-image:
    needs: config
    uses: ./.github/workflows/prepare-images.yml
    with:
      docker_image_tag:           ${{ needs.config.outputs.docker_image_tag }}
      need_linux_image_rebuild:   ${{ needs.config.outputs.need_linux_image_rebuild == 'true' }}
      need_windows_vcpkg_rebuild: ${{ needs.config.outputs.need_windows_vcpkg_rebuild == 'true' }}
      vcpkg_version:              ${{ needs.config.outputs.vcpkg_version }}
    secrets: inherit

  versioning-and-release-url:
    needs: config
    uses: ./.github/workflows/versioning-release.yml
    with:
      app_version:       ${{ needs.config.outputs.app_version }}
      release_tag:       ${{ needs.config.outputs.release_tag }}
      upload_artifacts:  ${{ needs.config.outputs.upload_artifacts == 'true' }}
    secrets:
      BUILD_MACHINE_TOKEN: ${{ secrets.BUILD_MACHINE_TOKEN }}

  windows-build-test:
    if: ${{ needs.config.outputs.build_enable_windows == 'true' }}
    needs: [ config, prepare-image ]
    uses: ./.github/workflows/build-test-windows.yml
    with:
      autotest_data_s3_url:  ${{ needs.config.outputs.autotest_data_s3_url }}
      full_config_build:     ${{ needs.config.outputs.full_config_build == 'true' }}
      internal_build:        ${{ needs.config.outputs.internal_build == 'true' }}
      upload_artifacts:      ${{ needs.config.outputs.upload_artifacts == 'true' }}
      upload_test_artifacts: ${{ needs.config.outputs.upload_test_artifacts == 'true' }}
      vcpkg_version:         ${{ needs.config.outputs.vcpkg_version }}
    secrets: inherit

  ubuntu-arm64-build-test:
    if: ${{ needs.config.outputs.build_enable_ubuntu_arm64 == 'true' }}
    needs: [ config, prepare-image ]
    uses: ./.github/workflows/build-test-ubuntu-arm64.yml
    with:
      app_version:           ${{ needs.config.outputs.app_version }}
      autotest_data_s3_url:  ${{ needs.config.outputs.autotest_data_s3_url }}
      docker_image_tag:      ${{ needs.config.outputs.docker_image_tag }}
      full_config_build:     ${{ needs.config.outputs.full_config_build == 'true' }}
      internal_build:        ${{ needs.config.outputs.internal_build == 'true' }}
      upload_artifacts:      ${{ needs.config.outputs.upload_artifacts == 'true' }}
      upload_test_artifacts: ${{ needs.config.outputs.upload_test_artifacts == 'true' }}
    secrets: inherit

  ubuntu-x64-build-test:
    if: ${{ needs.config.outputs.build_enable_ubuntu_x64 == 'true' }}
    needs: [ config, prepare-image ]
    uses: ./.github/workflows/build-test-ubuntu-x64.yml
    with:
      app_version:           ${{ needs.config.outputs.app_version }}
      autotest_data_s3_url:  ${{ needs.config.outputs.autotest_data_s3_url }}
      config_matrix:         ${{ needs.config.outputs.ubuntu_x64_config_matrix }}
      docker_image_tag:      ${{ needs.config.outputs.docker_image_tag }}
      full_config_build:     ${{ needs.config.outputs.full_config_build == 'true' }}
      internal_build:        ${{ needs.config.outputs.internal_build == 'true' }}
      upload_artifacts:      ${{ needs.config.outputs.upload_artifacts == 'true' }}
      upload_test_artifacts: ${{ needs.config.outputs.upload_test_artifacts == 'true' }}
    secrets: inherit

  fedora-build-test:
    if: ${{ needs.config.outputs.build_enable_fedora == 'true' }}
    needs: [ config, prepare-image ]
    uses: ./.github/workflows/build-test-fedora.yml
    with:
      app_version:           ${{ needs.config.outputs.app_version }}
      autotest_data_s3_url:  ${{ needs.config.outputs.autotest_data_s3_url }}
      docker_image_tag:      ${{ needs.config.outputs.docker_image_tag }}
      full_config_build:     ${{ needs.config.outputs.full_config_build == 'true' }}
      internal_build:        ${{ needs.config.outputs.internal_build == 'true' }}
      upload_artifacts:      ${{ needs.config.outputs.upload_artifacts == 'true' }}
      upload_test_artifacts: ${{ needs.config.outputs.upload_test_artifacts == 'true' }}
    secrets: inherit

  emscripten-build-test:
    if: ${{ needs.config.outputs.build_enable_emscripten == 'true' }}
    needs: [ config, prepare-image ]
    uses: ./.github/workflows/build-test-emscripten.yml
    with:
      docker_image_tag: ${{ needs.config.outputs.docker_image_tag }}

  macos-build-test:
    if: ${{ needs.config.outputs.build_enable_macos == 'true' }}
    needs: [ config ]
    uses: ./.github/workflows/build-test-macos.yml
    with:
      app_version:           ${{ needs.config.outputs.app_version }}
      autotest_data_s3_url:  ${{ needs.config.outputs.autotest_data_s3_url }}
      full_config_build:     ${{ needs.config.outputs.full_config_build == 'true' }}
      internal_build:        ${{ needs.config.outputs.internal_build == 'true' }}
      upload_artifacts:      ${{ needs.config.outputs.upload_artifacts == 'true' }}
      upload_test_artifacts: ${{ needs.config.outputs.upload_test_artifacts == 'true' }}
    secrets: inherit

  update-win-version:
    if: ${{ needs.config.outputs.upload_artifacts == 'true' && needs.config.outputs.build_enable_windows == 'true' }}
    needs:
      - config
      - windows-build-test
    timeout-minutes: 60
    runs-on: windows-2019
    env:
      vcpkg_version: ${{ needs.config.outputs.vcpkg_version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: true

      - name: Checkout Vcpkg ${{env.vcpkg_version}}
        working-directory: C:\vcpkg
        run: |
          git fetch
          git checkout ${{env.vcpkg_version}}

      - name: Restore Vcpkg Cache
        uses: actions/cache@v4
        id: vcpkg-cache
        with:
          key: vcpkg-cache-${{env.vcpkg_version}}
          path: |
            C:\vcpkg\*

      - name: Update vcpkg packages
        run: |
          .\thirdparty\install.bat

      - name: Download Windows Binaries Archive
        uses: actions/download-artifact@v4
        with:
          pattern: WindowsArchive*
          merge-multiple: true

      - name: Extract Windows Binaries
        run: |
          tar -xvzf MREDist_Release.zip
          tar -xvzf MREDist_Debug.zip

      - name: Make Install Folder
        run: py -3.10 scripts\make_install_folder.py ${{ needs.config.outputs.app_version }}

      - name: Make Install Folder .NET
        run: py -3.10 scripts\make_install_folder_dotnet.py ${{ needs.config.outputs.app_version }}

      - name: Add msbuild to PATH
        uses: microsoft/setup-msbuild@v2

      - name: Archive Distribution
        run: py -3.10 scripts\zip_distribution.py install example_plugin MeshLibDist.zip

      - name: Test Example Plugin # call it after archive not to pack build results
        run: |
          msbuild -m example_plugin\example_plugin.sln -p:Configuration=Debug
          msbuild -m example_plugin\example_plugin.sln -p:Configuration=Release

      - name: Archive .NET Distribution
        run: py -3.10 scripts\zip_distribution.py install_dotnet MeshLibDotNetDist.zip

      - name: Upload Windows Distribution
        uses: actions/upload-artifact@v4
        with:
          name: DistributivesWin
          path: MeshLibDist.zip
          retention-days: 1

      - name: Upload .NET Distribution
        uses: actions/upload-artifact@v4
        with:
          name: DistributivesDotNet
          path: MeshLibDotNetDist.zip
          retention-days: 1

  create-nuget-package:
    if: ${{ needs.config.outputs.upload_artifacts == 'true' && needs.config.outputs.build_enable_windows == 'true' }}
    needs:
      - config
      - windows-build-test
    timeout-minutes: 20
    runs-on: windows-2022
    steps:
     - name: Checkout
       uses: actions/checkout@v4

     - name: Download .NET Binaries Archive
       uses: actions/download-artifact@v4
       with:
         pattern: DotNetArchive*
         merge-multiple: true

     - name: Extract Windows Binaries
       run: |
         tar -xvzf MREDist_DotNet.zip

     - name: Generate NuGet specification
       run: |
        echo ${{ needs.config.outputs.app_version }}
        py -3 scripts\generate_nuget_spec.py ${{ needs.config.outputs.app_version }}

     - name: Download NuGet Executable
       run:  curl https://dist.nuget.org/win-x86-commandline/latest/nuget.exe -o nuget.exe

     - name: Build NuGet package
       run: nuget.exe pack Package.nuspec -OutputFileNamesWithoutVersion

     - name: Upload NuGet Package
       uses: actions/upload-artifact@v4
       with:
         name: DistributivesNuGet
         path: MeshLib.nupkg
         retention-days: 1

  upload-distributions:
    if: ${{ !cancelled() && needs.config.outputs.upload_artifacts == 'true' }}
    timeout-minutes: 10
    runs-on: ubuntu-latest
    needs:
      - config
      - versioning-and-release-url
      - update-win-version
      - create-nuget-package
      - ubuntu-x64-build-test
      - ubuntu-arm64-build-test
      - fedora-build-test
      - emscripten-build-test
      - macos-build-test
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download All Developer Distributives
        uses: actions/download-artifact@v4
        with:
          pattern: Distributives*
          merge-multiple: true

      - name: Rename Distributives
        run: |
          shopt -s nullglob
          for PKG_FILE in MeshLibDist*.zip ; do
            mv "$PKG_FILE" "${PKG_FILE/Dist/Dist_${{ needs.config.outputs.app_version }}}"
          done
          for PKG_FILE in MeshLib*.nupkg ; do
            mv "$PKG_FILE" "${PKG_FILE/MeshLib/MeshLib_${{ needs.config.outputs.app_version }}}"
          done
          for PKG_FILE in meshlib*-dev.deb meshlib*-dev.rpm meshlib*.pkg; do
            mv "$PKG_FILE" "${PKG_FILE/meshlib/meshlib_${{ needs.config.outputs.app_version }}}"
          done

      - name: Upload Distributives
        run: |
          shopt -s nullglob
          PKG_FILES="MeshLibDist*.zip meshlib*-dev.deb meshlib*-dev.rpm meshlib*.pkg MeshLib*.nupkg"
          if [ -n "$(echo $PKG_FILES)" ] ; then
            echo ${{ secrets.BUILD_MACHINE_TOKEN }} | gh auth login --with-token
            gh release upload ${{ needs.config.outputs.release_tag }} $PKG_FILES --clobber
          fi

  test-distribution:
    if: ${{ needs.config.outputs.upload_artifacts == 'true' }}
    needs: [ config, versioning-and-release-url, upload-distributions ]
    uses: ./.github/workflows/test-distribution.yml
    with:
      release_id: ${{ needs.versioning-and-release-url.outputs.release_id }}

  update-dev-documentation:
    if: ${{ needs.config.outputs.upload_artifacts == 'true' || needs.config.outputs.update_doc == 'true' }}
    needs: [ config, ubuntu-arm64-build-test ]
    uses: ./.github/workflows/update-docs.yml
    with:
      output_folder: MeshLib/dev
    secrets: inherit

  test-pip-build:
    needs: config
    if: ${{ needs.config.outputs.test_pip_build == 'true' }}
    uses: ./.github/workflows/pip-build.yml

  collect-stats:
    timeout-minutes: 5
    runs-on: ubuntu-latest
    needs: [ upload-distributions ]
    if: always()
    continue-on-error: true
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download runners' system stats
        uses: actions/download-artifact@v4
        with:
          pattern: RunnerSysStats*
          merge-multiple: true

      - name: Collect stats
        env:
          CI_STATS_AUTH_TOKEN: ${{ secrets.CI_STATS_AUTH_TOKEN }}
          GIT_BRANCH: ${{ github.head_ref || github.ref_name }}
          GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: python3 scripts/devops/collect_ci_stats.py

  update-artifacts:
    timeout-minutes: 15
    runs-on: ubuntu-20.04
    needs: [ collect-stats ]
    if: always()
    continue-on-error: true
    steps:
        # all Windows
      - name: Delete Windows Debug Install Archive artifact
        uses: geekyeggo/delete-artifact@v5
        with:
          name: WindowsArchive*
          failOnError: false
        # .NET
      - name: Delete .NET Binaries Archive artifact
        uses: geekyeggo/delete-artifact@v5
        with:
          name: DotNetArchive*
          failOnError: false
      # Distributives
      - name: Delete Distribution
        uses: geekyeggo/delete-artifact@v5
        with:
          name: Distributives*
          failOnError: false

      - name: Delete Wheelhouse
        uses: geekyeggo/delete-artifact@v5
        with:
          name: Wheelhouse*
          failOnError: false

      - name: Delete runners' system stats
        uses: geekyeggo/delete-artifact@v5
        with:
          name: RunnerSysStats*
          failOnError: false

      # generate timing logs archive
      - name: Download Timing Logs Archive
        uses: actions/download-artifact@v4
        with:
          pattern: Timing_Logs*
          merge-multiple: true

      - name: Generate Timing Logs Archive
        run: |
          shopt -s nullglob
          if [ -n "$(echo *.csv)" ] ; then
            tar -cvzf time_log.tar.gz *.csv
          else
            # create empty tar archive
            # https://superuser.com/questions/448623/how-to-get-an-empty-tar-archive
            tar -czf time_log.tar.gz -T /dev/null
          fi

      - name: Upload Timing Logs Archive
        uses: actions/upload-artifact@v4
        with:
          name: time_log
          path: time_log.tar.gz
          retention-days: 1

      - name: Delete Timing Logs Source
        uses: geekyeggo/delete-artifact@v5
        with:
          name: Timing_Logs*
          failOnError: false

      - name: Delete Python Stubs
        uses: geekyeggo/delete-artifact@v5
        with:
          name: PythonStubs
          failOnError: false