macos-ci.yml

# Copyright (C) 2022 Roberto Rossini (roberros@uio.no)
# SPDX-License-Identifier: MIT

name: MacOS CI

on:
  push:
    branches: [ main ]
    paths:
      - ".github/workflows/macos-ci.yml"
      - "cmake/**"
      - "examples/**"
      - "src/**"
      - "test/**"
      - "CMakeLists.txt"
      - "conanfile.txt"
    tags:
      - 'v*.*.*'

  pull_request:
    paths:
      - ".github/workflows/macos-ci.yml"
      - "cmake/**"
      - "examples/**"
      - "src/**"
      - "test/**"
      - "CMakeLists.txt"
      - "conanfile.txt"

# https://stackoverflow.com/a/72408109
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  CCACHE_DIR: "${{ github.workspace }}/ccache-cache"
  CCACHE_COMPILERCHECK: "content"
  CCACHE_COMPRESSLEVEL: "1"
  CCACHE_MAXSIZE: "250M"
  CONAN_HOME: "${{ github.workspace }}/.conan2"
  HOMEBREW_NO_AUTO_UPDATE: "1"

defaults:
  run:
    shell: bash

jobs:
  matrix-factory:
    name: Generate job matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-result.outputs.result }}
    steps:
      - uses: actions/github-script@v6
        id: set-result
        with:
          script: |
            // Documentation
            // https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson
            // https://github.com/actions/runner/issues/982#issuecomment-809360765

            var includes = []

            includes.push({ compiler_name: 'apple-clang', compiler_version: '13', os: 'macos-11', conan: '2.0.*', cmake: '3.26.*', build_type: 'Release', developer_mode: 'OFF' })
            includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-12', conan: '2.0.*', cmake: '3.26.*', build_type: 'Release', developer_mode: 'OFF' })
            includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-13', conan: '2.0.*', cmake: '3.26.*', build_type: 'Release', developer_mode: 'OFF' })
            includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-13', conan: '2.0.*', cmake: '3.26.*', build_type: 'Debug',   developer_mode: 'OFF' })

            return { include: includes }

  cache-test-dataset:
    uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main

  build-project:
    needs: matrix-factory
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

    steps:
      - uses: actions/checkout@v4

      - name: Generate requirements.txt for pip
        run: |
          echo 'conan==${{ matrix.conan }}' > requirements.txt
          echo 'cmake==${{ matrix.cmake }}' >> requirements.txt

      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Detect number available CPUs
        run: |
          ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())')

          echo "CMAKE_BUILD_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV
          echo "CTEST_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV

      - name: Install build deps
        run: |
          pip install -r requirements.txt
          brew install ccache

      - name: Generate cache key
        id: cache-key
        run: |
          set -u

          os="${{ matrix.os }}"
          compiler="${{ matrix.compiler_name }}"
          compiler_version="${{ matrix.compiler_version }}"
          build_type="${{ matrix.build_type }}"

          conanfile_hash="${{ hashFiles('conanfile.txt') }}"
          workflow_hash="${{ hashFiles('.github/workflows/macos-ci.yml') }}"
          combined_hash="${{ hashFiles('conanfile.txt', '.github/workflows/macos-ci.yml') }}"

          # This can be used by to always update a cache entry (useful e.g. for ccache)
          current_date="$(date '+%s')"

          conan_key_prefix="conan-$os-$compiler-$compiler_version-$conanfile_hash-$build_type"
          ccache_key_prefix="ccache-$os-$compiler-$compiler_version-$conanfile_hash-$build_type"

          echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT
          echo "conan-restore-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT

          echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a $GITHUB_OUTPUT
          echo "ccache-restore-key=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT

      - name: Restore Conan cache
        id: cache-conan
        uses: actions/cache/restore@v3
        with:
          key: conan-${{ steps.cache-key.outputs.conan-restore-key }}
          path: ${{ env.CONAN_HOME }}

      - name: Configure Conan
        if: steps.cache-conan.outputs.cache-hit != 'true'
        run: |
          conan profile detect --force

      - name: Clean Conan cache (pre-build)
        if: steps.cache-conan.outputs.cache-hit != 'true'
        run: |
          conan cache clean "*" --build
          conan cache clean "*" --download
          conan cache clean "*" --source
          conan remove --confirm "*"

      - name: Install build dependencies
        run: |
          conan install .                                         \
             --build=missing                                      \
             -pr default                                          \
             -s "build_type=${{ matrix.build_type }}"             \
             -s "compiler=${{ matrix.compiler_name }}"            \
             -s "compiler.version=${{ matrix.compiler_version }}" \
             -s compiler.libcxx=libc++                            \
             -s compiler.cppstd=17                                \
             --output-folder="${{ github.workspace }}/build"

      - name: Clean Conan cache (post-build)
        if: steps.cache-conan.outputs.cache-hit != 'true'
        run: |
          conan cache clean "*" --build
          conan cache clean "*" --download
          conan cache clean "*" --source

      - name: Save Conan cache
        uses: actions/cache/save@v3
        if: steps.cache-conan.outputs.cache-hit != 'true'
        with:
          key: conan-${{ steps.cache-key.outputs.conan-key }}
          path: ${{ env.CONAN_HOME }}

      - name: Cache Ccache folder
        uses: actions/cache@v3
        with:
          key: ${{ steps.cache-key.outputs.ccache-key }}
          restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key }}
          path: ${{ env.CCACHE_DIR }}

      - name: Configure project
        run: |
          cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}          \
                -DCMAKE_PREFIX_PATH="${{ github.workspace }}/build"  \
                -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \
                -DHICTK_ENABLE_TESTING=ON                            \
                -DHICTK_BUILD_EXAMPLES=ON                            \
                -DOPT_ENABLE_CLANG_TIDY=OFF                          \
                -DOPT_ENABLE_CPPCHECK=OFF                            \
                -DHICTK_DOWNLOAD_TEST_DATASET=OFF                    \
                -DHICTK_ENABLE_GIT_VERSION_TRACKING=OFF              \
                -DCMAKE_INSTALL_PREFIX=dest                          \
                -S "${{ github.workspace }}"                         \
                -B "${{ github.workspace }}/build"

      - name: Build project
        run: cmake --build ${{ github.workspace }}/build

      - name: Package binaries
        run: |
          cmake --install build

          gtar -cf - -C dest/ bin |
          zstd -T0 -13 -o binaries.tar.zst

      - name: Package unit tests
        run: |
          gtar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst

      - name: Upload unit tests
        uses: actions/upload-artifact@v3
        with:
          name: >
            unit-tests-${{ matrix.os }}-
            ${{ matrix.compiler_name }}-
            ${{ matrix.compiler_version }}-
            ${{ matrix.build_type }}-
            ${{ matrix.developer_mode }}
          path: unit-tests.tar.zst
          if-no-files-found: error
          retention-days: 1

      - name: Upload binaries
        uses: actions/upload-artifact@v3
        with:
          name: >
            binaries-${{ matrix.os }}-
            ${{ matrix.compiler_name }}-
            ${{ matrix.compiler_version }}-
            ${{ matrix.build_type }}-
            ${{ matrix.developer_mode }}
          path: binaries.tar.zst
          if-no-files-found: error
          retention-days: 1

      - name: Print Ccache statistics
        run: ccache -s


  run-unit-tests:
    name: Run unit tests
    needs: [ matrix-factory, cache-test-dataset, build-project ]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Fix permissions
        run: |
          chown -R $(id -u):$(id -g) $PWD

      - name: Restore test dataset
        uses: actions/cache/restore@v3
        with:
          key: ${{ needs.cache-test-dataset.outputs.cache-key }}
          path: test/data/hictk_test_data.tar.xz
          fail-on-cache-miss: true

      - name: Download unit tests artifact
        uses: actions/download-artifact@v3
        with:
          name: >
            unit-tests-${{ matrix.os }}-
            ${{ matrix.compiler_name }}-
            ${{ matrix.compiler_version }}-
            ${{ matrix.build_type }}-
            ${{ matrix.developer_mode }}

      - name: Extract binaries test dataset
        run: |
          zstd -dcf unit-tests.tar.zst | gtar -xf -
          gtar -xf test/data/hictk_test_data.tar.xz

      - name: Generate requirements.txt for pip
        run: |
          echo 'cmake==${{ matrix.cmake }}' > requirements.txt

      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install test dependencies
        run: pip install -r requirements.txt

      - name: Run unit tests
        run: |
          ctest --test-dir build/   \
                --schedule-random   \
                --output-on-failure \
                --no-tests=error    \
                --timeout 180 2>&1 |
          head -n 1000

  run-integration-tests:
    name: Run integration tests
    needs: [ matrix-factory, cache-test-dataset, build-project ]
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}

    steps:
      - name: Checkout repo
        uses: actions/checkout@v4

      - name: Fix permissions
        run: |
          chown -R $(id -u):$(id -g) $PWD

      - name: Generate requirements.txt for pip
        run: echo 'cooler>=0.9.2' > requirements.txt

      - uses: actions/setup-python@v4
        with:
          python-version: '3.11'
          cache: 'pip'

      - name: Install test dependencies
        run: pip install -r requirements.txt

      - name: Cache hic_tools
        id: cache-hictools
        uses: actions/cache@v3
        with:
          key: hic_tools
          path: hic_tools.jar

      - name: Download hic_tools
        if: steps.cache-hictools.outputs.cache-hit != 'true'
        run: |
          curl -L 'https://github.com/aidenlab/HiCTools/releases/download/v3.30.00/hic_tools.3.30.00.jar' \
               -o hic_tools.jar

      - name: Restore test dataset
        uses: actions/cache/restore@v3
        with:
          key: ${{ needs.cache-test-dataset.outputs.cache-key }}
          path: test/data/hictk_test_data.tar.xz
          fail-on-cache-miss: true

      - name: Download binaries artifact
        uses: actions/download-artifact@v3
        with:
          name: >
            binaries-${{ matrix.os }}-
            ${{ matrix.compiler_name }}-
            ${{ matrix.compiler_version }}-
            ${{ matrix.build_type }}-
            ${{ matrix.developer_mode }}

      - name: Extract binaries test dataset
        run: |
          zstd -dcf binaries.tar.zst | tar -xf -
          tar -xf test/data/hictk_test_data.tar.xz

      - name: Test hictk balance
        run: |
          test/scripts/hictk_balance.sh bin/hictk hic_tools.jar

      - name: Test hictk dump chroms
        run: |
          test/scripts/hictk_dump_chroms.sh bin/hictk

      - name: Test hictk dump bins
        run: |
          test/scripts/hictk_dump_bins.sh bin/hictk

      - name: Test hictk dump resolutions
        run: |
          test/scripts/hictk_dump_resolutions.sh bin/hictk

      - name: Test hictk dump normalizations
        run: |
          test/scripts/hictk_dump_normalizations.sh bin/hictk

      - name: Test hictk dump cells
        run: |
          test/scripts/hictk_dump_cells.sh bin/hictk

      - name: Test hictk dump genome-wide
        run: |
          test/scripts/hictk_dump_gw.sh bin/hictk

      - name: Test hictk dump cis
        run: |
          test/scripts/hictk_dump_cis.sh bin/hictk

      - name: Test hictk dump trans
        run: |
          test/scripts/hictk_dump_trans.sh bin/hictk

      - name: Test hictk dump balanced
        run: |
          test/scripts/hictk_dump_balanced.sh bin/hictk

      - name: Test hictk convert (hic2cool)
        run: |
          test/scripts/hictk_convert_hic2cool.sh bin/hictk

      - name: Test hictk convert (cool2hic)
        run: |
          test/scripts/hictk_convert_cool2hic.sh bin/hictk hic_tools.jar

      - name: Test hictk load coo sorted
        run: |
          test/scripts/hictk_load_coo.sh bin/hictk sorted

      - name: Test hictk load coo unsorted
        run: |
          test/scripts/hictk_load_coo.sh bin/hictk unsorted

      - name: Test hictk load bg2 sorted
        run: |
          test/scripts/hictk_load_bg2.sh bin/hictk sorted

      - name: Test hictk load bg2 unsorted
        run: |
          test/scripts/hictk_load_bg2.sh bin/hictk unsorted

      - name: Test hictk load 4dn sorted
        run: |
          test/scripts/hictk_load_4dn.sh bin/hictk sorted

      - name: Test hictk load 4dn unsorted
        run: |
          test/scripts/hictk_load_4dn.sh bin/hictk unsorted

      - name: Test hictk merge
        run: |
          test/scripts/hictk_merge.sh bin/hictk

      - name: Test hictk validate
        run: |
          test/scripts/hictk_validate.sh bin/hictk

      - name: Test hictk zoomify
        run: |
          test/scripts/hictk_zoomify.sh bin/hictk

  macos-ci-status-check:
    name: Status Check (MacOS CI)
    if: ${{ always() }}
    runs-on: ubuntu-latest
    needs:
      - matrix-factory
      - cache-test-dataset
      - build-project
      - run-unit-tests
      - run-integration-tests

    steps:
      - name: Collect job results
        if: |
          needs.matrix-factory.result != 'success'     ||
          needs.cache-test-dataset.result != 'success' ||
          needs.build-project.result != 'success'      ||
          needs.run-unit-tests.result != 'success'     ||
          needs.run-integration-tests.result != 'success'
        run: exit 1