Skip to content
Ubuntu CI

Implement ICE balancing #491

Sign in to view logs

Implement ICE balancing

Implement ICE balancing #491

Workflow file for this run

.github/workflows/ubuntu-ci.yml at d8ad3df
# Copyright (C) 2022 Roberto Rossini (roberros@uio.no)
# SPDX-License-Identifier: MIT
name: Ubuntu CI
on:
push:
branches: [ main ]
paths:
- ".github/workflows/ubuntu-ci.yml"
- "cmake/**"
- "examples/**"
- "src/**"
- "test/**"
- "CMakeLists.txt"
- "conanfile.txt"
tags:
- 'v*.*.*'
pull_request:
paths:
- ".github/workflows/ubuntu-ci.yml"
- "cmake/**"
- "examples/**"
- "src/**"
- "test/**"
- "CMakeLists.txt"
- "conanfile.txt"
# https://stackoverflow.com/a/72408109
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
defaults:
run:
shell: bash
jobs:
matrix-factory:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-result.outputs.result }}
steps:
- name: Checkout repo
uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.after }}
- name: Detect CI type
id: ci-type
run: |
if git log --format=%B -n 1 ${{ github.event.after }} | grep -qF '[ci full]'; then
echo "type=full" >> $GITHUB_OUTPUT
else
echo "type=short" >> $GITHUB_OUTPUT
fi
- name: Generate matrix
uses: actions/github-script@v6
id: set-result
with:
script: |
// Documentation
// https://docs.github.com/en/actions/learn-github-actions/contexts#fromjson
// https://github.com/actions/runner/issues/982#issuecomment-809360765
var ci_short = "${{ steps.ci-type.outputs.type }}" === "short"
var includes = []
// Debug builds (short CI)
includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'gcc-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' })
includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' })
// Release builds (short CI)
includes.push({ compiler: 'gcc-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' })
includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' })
if (ci_short) {
return { include: includes }
}
// Debug builds (long CI)
includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'gcc-10', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'gcc-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' })
includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' })
includes.push({ compiler: 'clang-14', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Debug', developer_mode: 'ON' })
// Release builds (long CI)
includes.push({ compiler: 'gcc-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
includes.push({ compiler: 'gcc-9', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
includes.push({ compiler: 'gcc-10', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' })
includes.push({ compiler: 'gcc-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' })
includes.push({ compiler: 'clang-8', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-9', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-10', os: 'ubuntu-20.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-11', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
includes.push({ compiler: 'clang-12', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' })
includes.push({ compiler: 'clang-13', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' })
includes.push({ compiler: 'clang-14', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'ON' })
// Make sure project builds with CMake 3.25
includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Ninja', cmake: '3.25.2', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
// Make sure project builds with make
includes.push({ compiler: 'clang-15', os: 'ubuntu-22.04', generator: 'Unix Makefiles', cmake: '3.26.*', conan: '2.0.*', build_type: 'Release', developer_mode: 'OFF' })
return { include: includes }
cache-test-dataset:
uses: paulsengroup/hictk/.github/workflows/cache-test-dataset.yml@main
build-project:
needs: matrix-factory
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}
container:
image: ghcr.io/paulsengroup/ci-docker-images/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest
options: "--user=root"
env:
CCACHE_DIR: "/opt/ccache-cache"
CCACHE_COMPILERCHECK: "content"
CCACHE_COMPRESSLEVEL: "1"
CCACHE_MAXSIZE: "350M"
CONAN_HOME: "/opt/conan/"
steps:
- uses: actions/checkout@v4
- name: Fix permissions
run: |
chown -R $(id -u):$(id -g) $PWD
- name: Check build deps are up-to-date
id: build-deps-outdated
run: |
pattern='[[:digit:]]\+\.[[:digit:]]\+\.[[:digit:]]\+$'
flag=false
if [[ $(cmake --version | grep -o "$pattern") != ${{ matrix.cmake }} ||
$(conan --version | grep -o "$pattern") != ${{ matrix.conan }} ]]; then
flag=true
fi
echo "outdated=$flag" >> $GITHUB_OUTPUT
- name: Update build deps
if: ${{ steps.buld-deps-outdated.outputs.outdated }}
run: |
apt-get update
apt-get install -y --no-install-recommends python3-pip
python3 -m pip install \
"cmake==${{ matrix.cmake }}" \
"conan==${{ matrix.conan }}"
- name: Generate cache keys
id: cache-key
run: |
set -u
os="${{ matrix.os }}"
compiler="${{ matrix.compiler }}"
generator="${{ matrix.generator }}"
build_type="${{ matrix.build_type }}"
dev_mode="${{ matrix.developer_mode }}"
conanfile_hash="${{ hashFiles('conanfile.txt') }}"
workflow_hash="${{ hashFiles('.github/workflows/ubuntu-ci.yml') }}"
combined_hash="${{ hashFiles('conanfile.txt', '.github/workflows/ubuntu-ci.yml') }}"
# This can be used by to always update a cache entry (useful e.g. for ccache)
current_date="$(date '+%s')"
conan_key_prefix="conan-$os-$compiler-$conanfile_hash-$build_type-$generator-$dev_mode"
ccache_key_prefix="ccache-$os-$compiler-$conanfile_hash-$build_type-$generator-$dev_mode"
echo "conan-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT
echo "conan-restore-key=$conan_key_prefix" | tee -a $GITHUB_OUTPUT
echo "ccache-key=${ccache_key_prefix}-${current_date}" | tee -a $GITHUB_OUTPUT
echo "ccache-restore-key=$ccache_key_prefix" | tee -a $GITHUB_OUTPUT
- name: Restore Conan cache
id: cache-conan
uses: actions/cache/restore@v3
with:
key: conan-${{ steps.cache-key.outputs.conan-restore-key }}
path: ${{ env.CONAN_HOME }}
- name: Clean Conan cache (pre-build)
if: steps.cache-conan.outputs.cache-hit != 'true'
run: |
conan cache clean "*" --build
conan cache clean "*" --download
conan cache clean "*" --source
conan remove --confirm "*"
- name: Install build dependencies
run: |
conan install . \
--build=missing \
-pr:b="$CONAN_DEFAULT_PROFILE_PATH" \
-pr:h="$CONAN_DEFAULT_PROFILE_PATH" \
-s build_type="${{ matrix.build_type }}" \
-s compiler.libcxx=libstdc++11 \
-s compiler.cppstd=17 \
--output-folder=build
- name: Clean Conan cache (post-build)
if: steps.cache-conan.outputs.cache-hit != 'true'
run: |
conan cache clean "*" --build
conan cache clean "*" --download
conan cache clean "*" --source
- name: Save Conan cache
uses: actions/cache/save@v3
if: steps.cache-conan.outputs.cache-hit != 'true'
with:
key: conan-${{ steps.cache-key.outputs.conan-key }}
path: ${{ env.CONAN_HOME }}
- name: Cache Ccache folder
uses: actions/cache@v3
with:
key: ${{ steps.cache-key.outputs.ccache-key }}
restore-keys: ${{ steps.cache-key.outputs.ccache-restore-key }}
path: ${{ env.CCACHE_DIR }}
- name: Configure project
run: |
cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
-DCMAKE_PREFIX_PATH="$PWD/build" \
-DENABLE_DEVELOPER_MODE=${{ matrix.developer_mode }} \
-DOPT_ENABLE_CLANG_TIDY=OFF \
-DOPT_ENABLE_CPPCHECK=OFF \
-DHICTK_ENABLE_TESTING=ON \
-DHICTK_BUILD_EXAMPLES=ON \
-DHICTK_DOWNLOAD_TEST_DATASET=OFF \
-DHICTK_ENABLE_GIT_VERSION_TRACKING=OFF \
-DCMAKE_INSTALL_PREFIX=dest \
-S . \
-B build
- name: Build project
run: cmake --build build -j $(nproc)
- name: Package binaries
run: |
cmake --install build
tar -cf - -C dest/ bin |
zstd -T0 -13 -o binaries.tar.zst
- name: Package unit tests
run: |
tar -cf - build/ | zstd -T0 -13 -o unit-tests.tar.zst
- name: Upload unit tests
uses: actions/upload-artifact@v3
with:
name: >
unit-tests-${{ matrix.os }}-
${{ matrix.compiler }}-
${{ matrix.generator }}-
${{ matrix.build_type }}-
${{ matrix.developer_mode }}
path: unit-tests.tar.zst
if-no-files-found: error
retention-days: 1
- name: Upload binaries
uses: actions/upload-artifact@v3
with:
name: >
binaries-${{ matrix.os }}-
${{ matrix.compiler }}-
${{ matrix.generator }}-
${{ matrix.build_type }}-
${{ matrix.developer_mode }}
path: binaries.tar.zst
if-no-files-found: error
retention-days: 1
- name: Print Ccache statistics
run: ccache -s
run-unit-tests:
name: Run unit tests
needs: [ matrix-factory, cache-test-dataset, build-project ]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}
container:
image: ghcr.io/paulsengroup/ci-docker-images/modle/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest
options: "--user=root"
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Fix permissions
run: |
chown -R $(id -u):$(id -g) $PWD
- name: Restore test dataset
uses: actions/cache/restore@v3
with:
key: ${{ needs.cache-test-dataset.outputs.cache-key }}
path: test/data/hictk_test_data.tar.xz
fail-on-cache-miss: true
- name: Download unit tests artifact
uses: actions/download-artifact@v3
with:
name: >
unit-tests-${{ matrix.os }}-
${{ matrix.compiler }}-
${{ matrix.generator }}-
${{ matrix.build_type }}-
${{ matrix.developer_mode }}
- name: Extract binaries and test dataset
run: |
zstd -dcf unit-tests.tar.zst | tar -xf -
tar -xf test/data/hictk_test_data.tar.xz
- name: Run unit tests
run: |
ctest --test-dir build/ \
--schedule-random \
--output-on-failure \
--no-tests=error \
--timeout 200 \
-j $(nproc) |&
head -n 1000
run-integration-tests:
name: Run integration tests
needs: [ matrix-factory, cache-test-dataset, build-project ]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.matrix-factory.outputs.matrix) }}
container:
image: ghcr.io/paulsengroup/ci-docker-images/modle/${{ matrix.os }}-cxx-${{ matrix.compiler }}:latest
options: "--user=root"
steps:
- name: Checkout repo
uses: actions/checkout@v4
- name: Fix permissions
run: |
chown -R $(id -u):$(id -g) $PWD
- name: Install test dependencies
run: |
apt-get update
apt-get install -y --no-install-recommends default-jdk python3-pip xz-utils
python3 -m pip install 'cooler>=0.9.2'
- name: Cache hic_tools
id: cache-hictools
uses: actions/cache@v3
with:
key: hic_tools
path: hic_tools.jar
- name: Download hic_tools
if: steps.cache-hictools.outputs.cache-hit != 'true'
run: |
apt-get install -y --no-install-recommends curl
curl -L 'https://github.com/aidenlab/HiCTools/releases/download/v3.30.00/hic_tools.3.30.00.jar' \
-o hic_tools.jar
- name: Restore test dataset
uses: actions/cache/restore@v3
with:
key: ${{ needs.cache-test-dataset.outputs.cache-key }}
path: test/data/hictk_test_data.tar.xz
fail-on-cache-miss: true
- name: Download binaries artifact
uses: actions/download-artifact@v3
with:
name: >
binaries-${{ matrix.os }}-
${{ matrix.compiler }}-
${{ matrix.generator }}-
${{ matrix.build_type }}-
${{ matrix.developer_mode }}
- name: Extract binaries and test dataset
run: |
zstd -dcf binaries.tar.zst | tar -xf -
tar -xf test/data/hictk_test_data.tar.xz
- name: Test hictk balance
run: |
test/scripts/hictk_balance.sh bin/hictk hic_tools.jar
- name: Test hictk dump chroms
run: |
test/scripts/hictk_dump_chroms.sh bin/hictk
- name: Test hictk dump bins
run: |
test/scripts/hictk_dump_bins.sh bin/hictk
- name: Test hictk dump resolutions
run: |
test/scripts/hictk_dump_resolutions.sh bin/hictk
- name: Test hictk dump normalizations
run: |
test/scripts/hictk_dump_normalizations.sh bin/hictk
- name: Test hictk dump cells
run: |
test/scripts/hictk_dump_cells.sh bin/hictk
- name: Test hictk dump genome-wide
run: |
test/scripts/hictk_dump_gw.sh bin/hictk
- name: Test hictk dump cis
run: |
test/scripts/hictk_dump_cis.sh bin/hictk
- name: Test hictk dump trans
run: |
test/scripts/hictk_dump_trans.sh bin/hictk
- name: Test hictk dump balanced
run: |
test/scripts/hictk_dump_balanced.sh bin/hictk
- name: Test hictk convert (hic2cool)
run: |
test/scripts/hictk_convert_hic2cool.sh bin/hictk
- name: Test hictk convert (cool2hic)
run: |
test/scripts/hictk_convert_cool2hic.sh bin/hictk hic_tools.jar
- name: Test hictk load coo sorted
run: |
test/scripts/hictk_load_coo.sh bin/hictk sorted
- name: Test hictk load coo unsorted
run: |
test/scripts/hictk_load_coo.sh bin/hictk unsorted
- name: Test hictk load bg2 sorted
run: |
test/scripts/hictk_load_bg2.sh bin/hictk sorted
- name: Test hictk load bg2 unsorted
run: |
test/scripts/hictk_load_bg2.sh bin/hictk unsorted
- name: Test hictk load 4dn sorted
run: |
test/scripts/hictk_load_4dn.sh bin/hictk sorted
- name: Test hictk load 4dn unsorted
run: |
test/scripts/hictk_load_4dn.sh bin/hictk unsorted
- name: Test hictk merge
run: |
test/scripts/hictk_merge.sh bin/hictk
- name: Test hictk validate
run: |
test/scripts/hictk_validate.sh bin/hictk
- name: Test hictk zoomify
run: |
test/scripts/hictk_zoomify.sh bin/hictk
ubuntu-ci-status-check:
name: Status Check (Ubuntu CI)
if: ${{ always() }}
runs-on: ubuntu-latest
needs:
- matrix-factory
- cache-test-dataset
- build-project
- run-unit-tests
- run-integration-tests
steps:
- name: Collect job results
if: |
needs.matrix-factory.result != 'success' ||
needs.cache-test-dataset.result != 'success' ||
needs.build-project.result != 'success' ||
needs.run-unit-tests.result != 'success' ||
needs.run-integration-tests.result != 'success'
run: exit 1