Implement hictk fix-mcool #507
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (C) 2022 Roberto Rossini (roberros@uio.no) | |
| # SPDX-License-Identifier: MIT | |
| name: Ubuntu CI | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - ".github/workflows/ubuntu-ci.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.txt" | |
| tags: | |
| - 'v*.*.*' | |
| pull_request: | |
| paths: | |
| - ".github/workflows/ubuntu-ci.yml" | |
| - "cmake/**" | |
| - "examples/**" | |
| - "src/**" | |
| - "test/**" | |
| - "CMakeLists.txt" | |
| - "conanfile.txt" | |
| # https://stackoverflow.com/a/72408109 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| matrix-factory: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-result.outputs.result }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| ref: ${{ github.event.after }} | |
| - name: Detect CI type | |
| id: ci-type | |
| run: | | |
| if git log --format=%B -n 1 ${{ github.event.after }} | grep -qF '[ci full]'; then | |
| echo "type=full" >> $GITHUB_OUTPUT | |
| else | |
| echo "type=short" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Generate matrix | |
| uses: actions/github-script@v6 | |
| id: set-result | |
| with: | |
| script: | | |
| // Documentation | |
| // https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson | |
| // https://github.com/actions/runner/issues/982#issuecomment-809360765 | |
| var ci_short = "${{ steps.ci-type.outputs.type }}" === "short" | |
| var includes = [] | |
| // Debug builds (short CI) | |
| includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'gcc-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' }) | |
| // Release builds (short CI) | |
| includes.push({ compiler: 'gcc-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' }) | |
| if (ci_short) { | |
| return { include: includes } | |
| } | |
| // Debug builds (long CI) | |
| includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'gcc-10', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'gcc-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'clang-14', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' }) | |
| // Release builds (long CI) | |
| includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'gcc-10', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'gcc-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' }) | |
| includes.push({ compiler: 'clang-14', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' }) | |
| // Make sure project builds with CMake 3.25 | |
| includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.25.2', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| // Make sure project builds with make | |
| includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Unix Makefiles', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' }) | |
| return { include: includes } | |
| cache-test-dataset: | |
| uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main | |
| build-project: | |
| needs: matrix-factory | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| container: | |
| image: ghcr.io/paulsengroup/ci-docker-images/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest | |
| options: "--user=root" | |
| env: | |
| CCACHE_DIR: "/opt/ccache-cache" | |
| CCACHE_COMPILERCHECK: "content" | |
| CCACHE_COMPRESSLEVEL: "1" | |
| CCACHE_MAXSIZE: "350M" | |
| CONAN_HOME: "/opt/conan/" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Fix permissions | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| - name: Check build deps are up-to-date | |
| id: build-deps-outdated | |
| run: | | |
| pattern='[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$' | |
| flag=false | |
| if [[ $(cmake --version | grep -o "$pattern") != ${{ matrix.cmake }} || | |
| $(conan --version | grep -o "$pattern") != ${{ matrix.conan }} ]]; then | |
| flag=true | |
| fi | |
| echo "outdated=$flag" >> $GITHUB_OUTPUT | |
| - name: Update build deps | |
| if: ${{ steps.buld-deps-outdated.outputs.outdated }} | |
| run: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends python3-pip | |
| python3 -m pip install \ | |
| "cmake==${{ matrix.cmake }}" \ | |
| "conan==${{ matrix.conan }}" | |
| - name: Generate cache keys | |
| id: cache-key | |
| run: | | |
| set -u | |
| os="${{ matrix.os }}" | |
| compiler="${{ matrix.compiler }}" | |
| generator="${{ matrix.generator }}" | |
| build_type="${{ matrix.build_type }}" | |
| dev_mode="${{ matrix.developer_mode }}" | |
| conanfile_hash="${{ hashFiles('conanfile.txt') }}" | |
| workflow_hash="${{ hashFiles('.github/workflows/ubuntu-ci.yml') }}" | |
| combined_hash="${{ hashFiles('conanfile.txt', '.github/workflows/ubuntu-ci.yml') }}" | |
| # This can be used by to always update a cache entry (useful e.g. for ccache) | |
| current_date="$(date '+%s')" | |
| conan_key_prefix="conan-$os-$compiler-$conanfile_hash-$build_type-$generator-$dev_mode" | |
| ccache_key_prefix="ccache-$os-$compiler-$conanfile_hash-$build_type-$generator-$dev_mode" | |
| echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT | |
| echo "conan-restore-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT | |
| echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a $GITHUB_OUTPUT | |
| echo "ccache-restore-key=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT | |
| - name: Restore Conan cache | |
| id: cache-conan | |
| uses: actions/cache/restore@v3 | |
| with: | |
| key: conan-${{ steps.cache-key.outputs.conan-restore-key }} | |
| path: ${{ env.CONAN_HOME }} | |
| - name: Clean Conan cache (pre-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| conan remove --confirm "*" | |
| - name: Install build dependencies | |
| run: | | |
| conan install . \ | |
| --build=missing \ | |
| -pr:b="$CONAN_DEFAULT_PROFILE_PATH" \ | |
| -pr:h="$CONAN_DEFAULT_PROFILE_PATH" \ | |
| -s build_type="${{ matrix.build_type }}" \ | |
| -s compiler.libcxx=libstdc++11 \ | |
| -s compiler.cppstd=17 \ | |
| --output-folder=build | |
| - name: Clean Conan cache (post-build) | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| run: | | |
| conan cache clean "*" --build | |
| conan cache clean "*" --download | |
| conan cache clean "*" --source | |
| - name: Save Conan cache | |
| uses: actions/cache/save@v3 | |
| if: steps.cache-conan.outputs.cache-hit != 'true' | |
| with: | |
| key: conan-${{ steps.cache-key.outputs.conan-key }} | |
| path: ${{ env.CONAN_HOME }} | |
| - name: Cache Ccache folder | |
| uses: actions/cache@v3 | |
| with: | |
| key: ${{ steps.cache-key.outputs.ccache-key }} | |
| restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key }} | |
| path: ${{ env.CCACHE_DIR }} | |
| - name: Configure project | |
| run: | | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCMAKE_PREFIX_PATH="$PWD/build" \ | |
| -DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \ | |
| -DOPT_ENABLE_CLANG_TIDY=OFF \ | |
| -DOPT_ENABLE_CPPCHECK=OFF \ | |
| -DHICTK_ENABLE_TESTING=ON \ | |
| -DHICTK_BUILD_EXAMPLES=ON \ | |
| -DHICTK_DOWNLOAD_TEST_DATASET=OFF \ | |
| -DHICTK_ENABLE_GIT_VERSION_TRACKING=OFF \ | |
| -DCMAKE_INSTALL_PREFIX=dest \ | |
| -S . \ | |
| -B build | |
| - name: Build project | |
| run: cmake --build build -j $(nproc) | |
| - name: Package binaries | |
| run: | | |
| cmake --install build | |
| tar -cf - -C dest/ bin | | |
| zstd -T0 -13 -o binaries.tar.zst | |
| - name: Package unit tests | |
| run: | | |
| tar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst | |
| - name: Upload unit tests | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: > | |
| unit-tests-${{ matrix.os }}- | |
| ${{ matrix.compiler }}- | |
| ${{ matrix.generator }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| path: unit-tests.tar.zst | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Upload binaries | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: > | |
| binaries-${{ matrix.os }}- | |
| ${{ matrix.compiler }}- | |
| ${{ matrix.generator }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| path: binaries.tar.zst | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - name: Print Ccache statistics | |
| run: ccache -s | |
| run-unit-tests: | |
| name: Run unit tests | |
| needs: [ matrix-factory, cache-test-dataset, build-project ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| container: | |
| image: ghcr.io/paulsengroup/ci-docker-images/modle/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest | |
| options: "--user=root" | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Fix permissions | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| - name: Restore test dataset | |
| uses: actions/cache/restore@v3 | |
| with: | |
| key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
| path: test/data/hictk_test_data.tar.xz | |
| fail-on-cache-miss: true | |
| - name: Download unit tests artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: > | |
| unit-tests-${{ matrix.os }}- | |
| ${{ matrix.compiler }}- | |
| ${{ matrix.generator }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| - name: Extract binaries and test dataset | |
| run: | | |
| zstd -dcf unit-tests.tar.zst | tar -xf - | |
| tar -xf test/data/hictk_test_data.tar.xz | |
| - name: Run unit tests | |
| run: | | |
| ctest --test-dir build/ \ | |
| --schedule-random \ | |
| --output-on-failure \ | |
| --no-tests=error \ | |
| --timeout 200 \ | |
| -j $(nproc) |& | |
| head -n 1000 | |
| run-integration-tests: | |
| name: Run integration tests | |
| needs: [ matrix-factory, cache-test-dataset, build-project ] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }} | |
| container: | |
| image: ghcr.io/paulsengroup/ci-docker-images/modle/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest | |
| options: "--user=root" | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - name: Fix permissions | |
| run: | | |
| chown -R $(id -u):$(id -g) $PWD | |
| - name: Install test dependencies | |
| run: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends default-jdk python3-pip xz-utils | |
| python3 -m pip install 'cooler>=0.9.2' | |
| - name: Cache hic_tools | |
| id: cache-hictools | |
| uses: actions/cache@v3 | |
| with: | |
| key: hic_tools | |
| path: hic_tools.jar | |
| - name: Download hic_tools | |
| if: steps.cache-hictools.outputs.cache-hit != 'true' | |
| run: | | |
| apt-get install -y --no-install-recommends curl | |
| curl -L 'https://github.com/aidenlab/HiCTools/releases/download/v3.30.00/hic_tools.3.30.00.jar' \ | |
| -o hic_tools.jar | |
| - name: Restore test dataset | |
| uses: actions/cache/restore@v3 | |
| with: | |
| key: ${{ needs.cache-test-dataset.outputs.cache-key }} | |
| path: test/data/hictk_test_data.tar.xz | |
| fail-on-cache-miss: true | |
| - name: Download binaries artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: > | |
| binaries-${{ matrix.os }}- | |
| ${{ matrix.compiler }}- | |
| ${{ matrix.generator }}- | |
| ${{ matrix.build_type }}- | |
| ${{ matrix.developer_mode }} | |
| - name: Extract binaries and test dataset | |
| run: | | |
| zstd -dcf binaries.tar.zst | tar -xf - | |
| tar -xf test/data/hictk_test_data.tar.xz | |
| - name: Test hictk balance | |
| run: | | |
| test/scripts/hictk_balance.sh bin/hictk hic_tools.jar | |
| - name: Test hictk convert (hic2cool) | |
| run: | | |
| test/scripts/hictk_convert_hic2cool.sh bin/hictk | |
| - name: Test hictk convert (cool2hic) | |
| run: | | |
| test/scripts/hictk_convert_cool2hic.sh bin/hictk hic_tools.jar | |
| - name: Test hictk dump chroms | |
| run: | | |
| test/scripts/hictk_dump_chroms.sh bin/hictk | |
| - name: Test hictk dump bins | |
| run: | | |
| test/scripts/hictk_dump_bins.sh bin/hictk | |
| - name: Test hictk dump resolutions | |
| run: | | |
| test/scripts/hictk_dump_resolutions.sh bin/hictk | |
| - name: Test hictk dump normalizations | |
| run: | | |
| test/scripts/hictk_dump_normalizations.sh bin/hictk | |
| - name: Test hictk dump cells | |
| run: | | |
| test/scripts/hictk_dump_cells.sh bin/hictk | |
| - name: Test hictk dump genome-wide | |
| run: | | |
| test/scripts/hictk_dump_gw.sh bin/hictk | |
| - name: Test hictk dump cis | |
| run: | | |
| test/scripts/hictk_dump_cis.sh bin/hictk | |
| - name: Test hictk dump trans | |
| run: | | |
| test/scripts/hictk_dump_trans.sh bin/hictk | |
| - name: Test hictk dump balanced | |
| run: | | |
| test/scripts/hictk_dump_balanced.sh bin/hictk | |
| - name: Test hictk fix-mcool | |
| run: | | |
| test/scripts/hictk_fix_mcool.sh bin/hictk | |
| - name: Test hictk load coo sorted | |
| run: | | |
| test/scripts/hictk_load_coo.sh bin/hictk sorted | |
| - name: Test hictk load coo unsorted | |
| run: | | |
| test/scripts/hictk_load_coo.sh bin/hictk unsorted | |
| - name: Test hictk load bg2 sorted | |
| run: | | |
| test/scripts/hictk_load_bg2.sh bin/hictk sorted | |
| - name: Test hictk load bg2 unsorted | |
| run: | | |
| test/scripts/hictk_load_bg2.sh bin/hictk unsorted | |
| - name: Test hictk load 4dn sorted | |
| run: | | |
| test/scripts/hictk_load_4dn.sh bin/hictk sorted | |
| - name: Test hictk load 4dn unsorted | |
| run: | | |
| test/scripts/hictk_load_4dn.sh bin/hictk unsorted | |
| - name: Test hictk merge | |
| run: | | |
| test/scripts/hictk_merge.sh bin/hictk | |
| - name: Test hictk validate | |
| run: | | |
| test/scripts/hictk_validate.sh bin/hictk | |
| - name: Test hictk zoomify | |
| run: | | |
| test/scripts/hictk_zoomify.sh bin/hictk | |
| ubuntu-ci-status-check: | |
| name: Status Check (Ubuntu CI) | |
| if: ${{ always() }} | |
| runs-on: ubuntu-latest | |
| needs: | |
| - matrix-factory | |
| - cache-test-dataset | |
| - build-project | |
| - run-unit-tests | |
| - run-integration-tests | |
| steps: | |
| - name: Collect job results | |
| if: | | |
| needs.matrix-factory.result != 'success' || | |
| needs.cache-test-dataset.result != 'success' || | |
| needs.build-project.result != 'success' || | |
| needs.run-unit-tests.result != 'success' || | |
| needs.run-integration-tests.result != 'success' | |
| run: exit 1 |