Implement ICE balancing #480
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2022 Roberto Rossini (roberros@uio.no) | |
| # SPDX-License-Identifier: MIT | |
| name: MacOS CI | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - ".github/workflows/macos-ci.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.txt" | |
| tags: | |
| - 'v*.*.*' | |
| pull_request: | |
| paths: | |
| - ".github/workflows/macos-ci.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.txt" | |
| # https://stackoverflow.com/a/72408109 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CCACHE_DIR: "${{ github.workspace }}/ccache-cache" | |
| CCACHE_COMPILERCHECK: "content" | |
| CCACHE_COMPRESSLEVEL: "1" | |
| CCACHE_MAXSIZE: "250M" | |
| CONAN_HOME: "${{ github.workspace }}/.conan2" | |
| HOMEBREW_NO_AUTO_UPDATE: "1" | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| matrix-factory: | |
| name: Generate job matrix | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-result.outputs.result }} | |
| steps: | |
| - uses: actions/github-script@v6 | |
| id: set-result | |
| with: | |
| script: | | |
| // Documentation | |
| // https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson | |
| // https://github.com/actions/runner/issues/982#issuecomment-809360765 | |
| var includes = [] | |
| includes.push({ compiler_name: 'apple-clang', compiler_version: '13', os: 'macos-11', conan: '2.0.*', cmake: '3.26.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-12', conan: '2.0.*', cmake: '3.26.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-13', conan: '2.0.*', cmake: '3.26.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler_name: 'apple-clang', compiler_version: '14', os: 'macos-13', conan: '2.0.*', cmake: '3.26.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| return { include: includes } | |
| cache-test-dataset: | |
| uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main | |
| build-project: | |
| needs: matrix-factory | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Generate requirements.txt for pip | |
| run: | | |
| echo 'conan==${{ matrix.conan }}' > requirements.txt | |
| echo 'cmake==${{ matrix.cmake }}' >> requirements.txt | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Detect number available CPUs | |
| run: | | |
| ncpus=$(python -c 'import multiprocessing as mp; print(mp.cpu_count())') | |
| echo "CMAKE_BUILD_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV | |
| echo "CTEST_PARALLEL_LEVEL=$ncpus" >> $GITHUB_ENV | |
| - name: Install build deps | |
| run: | | |
| pip install -r requirements.txt | |
| brew install ccache | |
| - name: Generate cache key | |
| id: cache-key | |
| run: | | |
| set -u | |
| os="${{ matrix.os }}" | |
| compiler="${{ matrix.compiler_name }}" | |
| compiler_version="${{ matrix.compiler_version }}" | |
| build_type="${{ matrix.build_type }}" | |
| conanfile_hash="${{ hashFiles('conanfile.txt') }}" | |
| workflow_hash="${{ hashFiles('.github/workflows/macos-ci.yml') }}" | |
| combined_hash="${{ hashFiles('conanfile.txt', '.github/workflows/macos-ci.yml') }}" | |
| # This can be used by to always update a cache entry (useful e.g. for ccache) | |
| current_date="$(date '+%s')" | |
| conan_key_prefix="conan-$os-$compiler-$compiler_version-$conanfile_hash-$build_type" | |
| ccache_key_prefix="ccache-$os-$compiler-$compiler_version-$conanfile_hash-$build_type" | |
| echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT | |
| echo "conan-restore-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT | |
| echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a $GITHUB_OUTPUT | |
| echo "ccache-restore-key=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT | |
| - name: Restore Conan cache | |
| id: cache-conan | |
| uses: actions/cache/restore@v3 | |
| with: | |
| key: conan-${{ steps.cache-key.outputs.conan-restore-key }} | |
| path: ${{ env.CONAN_HOME }} | |
| - name: Configure Conan | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan profile detect --force | |
| - name: Clean Conan cache (pre-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| conan remove --confirm "*" | |
| - name: Install build dependencies | |
| run: | | |
| conan install . \ | |
| --build=missing \ | |
| -pr default \ | |
| -s "build_type=${{ matrix.build_type }}" \ | |
| -s "compiler=${{ matrix.compiler_name }}" \ | |
| -s "compiler.version=${{ matrix.compiler_version }}" \ | |
| -s compiler.libcxx=libc++ \ | |
| -s compiler.cppstd=17 \ | |
| --output-folder="${{ github.workspace }}/build" | |
| - name: Clean Conan cache (post-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| - name: Save Conan cache | |
| uses: actions/cache/save@v3 | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| with: | |
| key: conan-${{ steps.cache-key.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }} | |
| - name: Cache Ccache folder | |
| uses: actions/cache@v3 | |
| with: | |
| key: ${{ steps.cache-key.outputs.ccache-key }} | |
| restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Configure project | |
| run: | | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCMAKE_PREFIX_PATH="${{ github.workspace }}/build" \ | |
| -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \ | |
| -DHICTK_ENABLE_TESTING=ON \ | |
| -DHICTK_BUILD_EXAMPLES=ON \ | |
| -DOPT_ENABLE_CLANG_TIDY=OFF \ | |
| -DOPT_ENABLE_CPPCHECK=OFF \ | |
| -DHICTK_DOWNLOAD_TEST_DATASET=OFF \ | |
| -DHICTK_ENABLE_GIT_VERSION_TRACKING=OFF \ | |
| -DCMAKE_INSTALL_PREFIX=dest \ | |
| -S "${{ github.workspace }}" \ | |
| -B "${{ github.workspace }}/build" | |
| - name: Build project | |
| run: cmake --build ${{ github.workspace }}/build | |
| - name: Package binaries | |
| run: | | |
| cmake --install build | |
| gtar -cf - -C dest/ bin | | |
| zstd -T0 -13 -o binaries.tar.zst | |
| - name: Package unit tests | |
| run: | | |
| gtar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst | |
| - name: Upload unit tests | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: > | |
| unit-tests-${{ matrix.os }}- | |
| ${{ matrix.compiler_name }}- | |
| ${{ matrix.compiler_version }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| path: unit-tests.tar.zst | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Upload binaries | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: > | |
| binaries-${{ matrix.os }}- | |
| ${{ matrix.compiler_name }}- | |
| ${{ matrix.compiler_version }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| path: binaries.tar.zst | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Print Ccache statistics | |
| run: ccache -s | |
| run-unit-tests: | |
| name: Run unit tests | |
| needs: [ matrix-factory, cache-test-dataset, build-project ] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Fix permissions | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| - name: Restore test dataset | |
| uses: actions/cache/restore@v3 | |
| with: | |
| key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
| path: test/data/hictk_test_data.tar.xz | |
| fail-on-cache-miss: true | |
| - name: Download unit tests artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: > | |
| unit-tests-${{ matrix.os }}- | |
| ${{ matrix.compiler_name }}- | |
| ${{ matrix.compiler_version }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| - name: Extract binaries test dataset | |
| run: | | |
| zstd -dcf unit-tests.tar.zst | gtar -xf - | |
| gtar -xf test/data/hictk_test_data.tar.xz | |
| - name: Generate requirements.txt for pip | |
| run: | | |
| echo 'cmake==${{ matrix.cmake }}' > requirements.txt | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install test dependencies | |
| run: pip install -r requirements.txt | |
| - name: Run unit tests | |
| run: | | |
| ctest --test-dir build/ \ | |
| --schedule-random \ | |
| --output-on-failure \ | |
| --no-tests=error \ | |
| --timeout 180 2>&1 | | |
| head -n 1000 | |
| run-integration-tests: | |
| name: Run integration tests | |
| needs: [ matrix-factory, cache-test-dataset, build-project ] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Fix permissions | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| - name: Generate requirements.txt for pip | |
| run: echo 'cooler>=0.9.2' > requirements.txt | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| - name: Install test dependencies | |
| run: pip install -r requirements.txt | |
| - name: Cache hic_tools | |
| id: cache-hictools | |
| uses: actions/cache@v3 | |
| with: | |
| key: hic_tools | |
| path: hic_tools.jar | |
| - name: Download hic_tools | |
| if: steps.cache-hictools.outputs.cache-hit != 'true' | |
| run: | | |
| curl -L 'https://github.com/aidenlab/HiCTools/releases/download/v3.30.00/hic_tools.3.30.00.jar' \ | |
| -o hic_tools.jar | |
| - name: Restore test dataset | |
| uses: actions/cache/restore@v3 | |
| with: | |
| key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
| path: test/data/hictk_test_data.tar.xz | |
| fail-on-cache-miss: true | |
| - name: Download binaries artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: > | |
| binaries-${{ matrix.os }}- | |
| ${{ matrix.compiler_name }}- | |
| ${{ matrix.compiler_version }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| - name: Extract binaries test dataset | |
| run: | | |
| zstd -dcf binaries.tar.zst | tar -xf - | |
| tar -xf test/data/hictk_test_data.tar.xz | |
| - name: Test hictk balance | |
| run: | | |
| test/scripts/hictk_balance.sh bin/hictk hic_tools.jar | |
| - name: Test hictk dump chroms | |
| run: | | |
| test/scripts/hictk_dump_chroms.sh bin/hictk | |
| - name: Test hictk dump bins | |
| run: | | |
| test/scripts/hictk_dump_bins.sh bin/hictk | |
| - name: Test hictk dump resolutions | |
| run: | | |
| test/scripts/hictk_dump_resolutions.sh bin/hictk | |
| - name: Test hictk dump normalizations | |
| run: | | |
| test/scripts/hictk_dump_normalizations.sh bin/hictk | |
| - name: Test hictk dump cells | |
| run: | | |
| test/scripts/hictk_dump_cells.sh bin/hictk | |
| - name: Test hictk dump genome-wide | |
| run: | | |
| test/scripts/hictk_dump_gw.sh bin/hictk | |
| - name: Test hictk dump cis | |
| run: | | |
| test/scripts/hictk_dump_cis.sh bin/hictk | |
| - name: Test hictk dump trans | |
| run: | | |
| test/scripts/hictk_dump_trans.sh bin/hictk | |
| - name: Test hictk dump balanced | |
| run: | | |
| test/scripts/hictk_dump_balanced.sh bin/hictk | |
| - name: Test hictk convert (hic2cool) | |
| run: | | |
| test/scripts/hictk_convert_hic2cool.sh bin/hictk | |
| - name: Test hictk convert (cool2hic) | |
| run: | | |
| test/scripts/hictk_convert_cool2hic.sh bin/hictk hic_tools.jar | |
| - name: Test hictk load coo sorted | |
| run: | | |
| test/scripts/hictk_load_coo.sh bin/hictk sorted | |
| - name: Test hictk load coo unsorted | |
| run: | | |
| test/scripts/hictk_load_coo.sh bin/hictk unsorted | |
| - name: Test hictk load bg2 sorted | |
| run: | | |
| test/scripts/hictk_load_bg2.sh bin/hictk sorted | |
| - name: Test hictk load bg2 unsorted | |
| run: | | |
| test/scripts/hictk_load_bg2.sh bin/hictk unsorted | |
| - name: Test hictk load 4dn sorted | |
| run: | | |
| test/scripts/hictk_load_4dn.sh bin/hictk sorted | |
| - name: Test hictk load 4dn unsorted | |
| run: | | |
| test/scripts/hictk_load_4dn.sh bin/hictk unsorted | |
| - name: Test hictk merge | |
| run: | | |
| test/scripts/hictk_merge.sh bin/hictk | |
| - name: Test hictk validate | |
| run: | | |
| test/scripts/hictk_validate.sh bin/hictk | |
| - name: Test hictk zoomify | |
| run: | | |
| test/scripts/hictk_zoomify.sh bin/hictk | |
| macos-ci-status-check: | |
| name: Status Check (MacOS CI) | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - matrix-factory | |
| - cache-test-dataset | |
| - build-project | |
| - run-unit-tests | |
| - run-integration-tests | |
| steps: | |
| - name: Collect job results | |
| if: | | |
| needs.matrix-factory.result != 'success' || | |
| needs.cache-test-dataset.result != 'success' || | |
| needs.build-project.result != 'success' || | |
| needs.run-unit-tests.result != 'success' || | |
| needs.run-integration-tests.result != 'success' | |
| run: exit 1 |