diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml new file mode 100644 index 000000000..b0ff1fb18 --- /dev/null +++ b/.gitlab/.gitlab-ci.yml @@ -0,0 +1,72 @@ +variables: + CUSTOM_CI_BUILDS_DIR: "/usr/workspace/$$USER/libROM_gitlab_runner" + + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: recursive + GIT_DEPTH: 1 + GIT_SUBMODULE_DEPTH: 1 + + DEFAULT_BRANCH: master + + ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} + ALLOC_QUEUE: pci + ALLOC_TIME: 15 + ALLOC_BANK: asccasc + + TEST_SCRIPT: .gitlab/run_tests.sh + + ON_LASSEN: "OFF" + #ON_DANE: "OFF" + +stages: + - allocate + - build + - release + +workflow: + rules: + # skip running branch pipelines if a MR is open for the branch + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == 'external_pull_request_event' + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +include: + # This include is required for LC with Gitlab 17+ + # Refer to https://hpc.llnl.gov/technical-bulletins/bulletin-568 + - project: 'lc-templates/id_tokens' + file: 'id_tokens.yml' + - .gitlab/runners/lassen.yml + - .gitlab/runners/dane.yml + +# Define actual CI jobs here: +dane_gcc_12_1_1: + variables: + MODULE_LIST: cmake gcc/12.1.1 + extends: .job_on_dane + +lassen_gcc_12_2_1: + variables: + MODULE_LIST: cmake/3.23.1 gcc/12.2.1 + extends: .job_on_lassen + +regtest_dane_gcc_12_1_1: + variables: + MODULE_LIST: cmake gcc/12.1.1 + LLNL_SLURM_SCHEDULER_PARAMETERS: --res=ci + MPIEXEC_EXECUTABLE: srun + MPIEXEC_PREFLAGS: "--cpu-bind=cores -v" + stage: allocate + rules: + #- if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_COMMIT_BRANCH != $DEFAULT_BRANCH + # when: manual + - when: manual + - allow_failure: true + tags: + - batch + extends: .on_dane + script: + - module load ${MODULE_LIST} + - ./regression_tests/run_regression_tests.sh diff --git a/.gitlab/run_tests.sh b/.gitlab/run_tests.sh new file mode 100755 index 000000000..dcb0d40ad --- /dev/null +++ b/.gitlab/run_tests.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +modules=${MODULE_LIST:-""} +mpiexec_executable=${MPIEXEC_EXECUTABLE:-"srun"} +# If using flux, append "run" after the flux executable path +if [[ "${mpiexec_executable}" == "flux" ]] +then + mpiexec_executable="$(which ${mpiexec_executable}) run" + flux jobs + flux resource list +else + mpiexec_executable="$(which ${mpiexec_executable})" +fi + +mpiexec_preflags=${MPIEXEC_PREFLAGS:-""} +host=$(hostname) +build_type=${BUILD_TYPE:-"Debug"} +# note: toolchain file here is relative to repo root dir +toolchain_file=${TOOLCHAIN_FILE:-"../cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake"} +use_mfem=${USE_MFEM:-"Off"} +use_gslib=${MFEM_USE_GSLIB:-"Off"} +librom_flags=${LIBROM_FLAGS:-""} + +basehost=${host//[[:digit:]]/} + +echo ${host} + +build_dir=build_${host}_${CI_PIPELINE_ID}_$(date +%F_%H_%M_%S) + +if [[ -n ${modules} ]] +then + module load ${modules} +fi + +# ---- setup googletest ---- +DEPS_DIR="${CI_BUILDS_DIR}/${basehost}_deps" +echo $DEPS_DIR +if [[ ! -d "${DEPS_DIR}" ]]; then + mkdir ${DEPS_DIR} && cd ${DEPS_DIR} + git clone https://github.com/google/googletest + cd googletest && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=./ .. && make && make install + cd ${CI_PROJECT_DIR} +fi +PATH=${DEPS_DIR}/googletest:${DEPS_DIR}/googletest/build:$PATH +echo ${PATH} +# -------------------------- + +#./scripts/compile.sh -d -r + +mkdir ${build_dir} +cd ${build_dir} +pwd + +cmake -DCMAKE_TOOLCHAIN_FILE=${toolchain_file} \ + -DCMAKE_BUILD_TYPE=${build_type} \ + -DMPIEXEC_EXECUTABLE=${mpiexec_executable} \ + -DMPIEXEC_PREFLAGS="${mpiexec_preflags}" \ + -DUSE_MFEM=${use_mfem} \ + -DMFEM_USE_GSLIB=${use_gslib} \ + -DENABLE_TESTS=ON \ + -DLIBROM_FLAGS="${librom_flags}" .. + +make -j + +ctest -VV --output-on-failure diff --git a/.gitlab/runners/dane.yml b/.gitlab/runners/dane.yml new file mode 100644 index 000000000..4b42371cb --- /dev/null +++ b/.gitlab/runners/dane.yml @@ -0,0 +1,54 @@ +.retry: + retry: + max: 2 + when: + - runner_system_failure + +.on_dane: + extends: + - .retry + tags: + - dane + - shell + rules: + - if: '$ON_DANE == "OFF"' + when: never + - if: $CI_PIPELINE_SOURCE == "push" + - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" + - if: $CI_COMMIT_BRANCH != $DEFAULT_BRANCH + - if: '$CI_JOB_NAME =~ /release_resources_dane/' + when: always + - when: on_success + +allocate_resources_dane: + variables: + GIT_STRATEGY: none + extends: + - .on_dane + stage: allocate + script: + - salloc -N 1 --reservation=ci -A ${ALLOC_BANK} --time=${ALLOC_TIME} --no-shell --job-name=${ALLOC_NAME} + +release_resources_dane: + variables: + GIT_STRATEGY: none + extends: + - .on_dane + stage: release + script: + - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) + - ([[ -n "${JOBID}" ]] && scancel ${JOBID}) + when: always + +.job_on_dane: + extends: .on_dane + stage: build + needs: ["allocate_resources_dane"] + variables: + MPIEXEC_EXECUTABLE: srun + MPIEXEC_PREFLAGS: "--cpu-bind=cores -v" + script: + - echo "JOB NAME ${ALLOC_NAME}" + - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) + - echo "SLURM ID ${JOBID}" + - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -N 1 -t ${ALLOC_TIME} -v --overlap ${TEST_SCRIPT} diff --git a/.gitlab/runners/lassen.yml b/.gitlab/runners/lassen.yml new file mode 100644 index 000000000..beedab917 --- /dev/null +++ b/.gitlab/runners/lassen.yml @@ -0,0 +1,30 @@ +.retry: + retry: + max: 2 + when: + - runner_system_failure + +.on_lassen: + extends: + - .retry + tags: + - lassen + - shell + rules: + - if: '$ON_LASSEN == "OFF"' + when: never + - if: $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" + - if: $CI_COMMIT_BRANCH == $DEFAULT_BRANCH + - when: on_success + +.job_on_lassen: + extends: .on_lassen + stage: build + needs: [] + variables: + MPIEXEC_EXECUTABLE: jsrun + MPIEXEC_PREFLAGS: "-a 1 -c 1 -g 1" + script: + - bsub -q ${ALLOC_QUEUE} -W ${ALLOC_TIME} -G ${ALLOC_BANK} -J ${ALLOC_NAME} -nnodes 1 -Is ${TEST_SCRIPT} diff --git a/unit_tests/CMakeLists.txt b/unit_tests/CMakeLists.txt index 3532b126b..51829801c 100644 --- a/unit_tests/CMakeLists.txt +++ b/unit_tests/CMakeLists.txt @@ -60,7 +60,9 @@ foreach(stem IN LISTS unit_test_stems) target_compile_features(test_${stem} PRIVATE cxx_std_11) target_compile_definitions(test_${stem} PRIVATE CAROM_HAS_GTEST) - add_test(NAME test_${stem} COMMAND test_${stem}) + # serial tests + set(test_command sh -c "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} test_${stem}") + add_test(NAME test_${stem} COMMAND ${test_command}) endforeach(stem) # Add parallel tests diff --git a/unit_tests/test_DEIM.cpp b/unit_tests/test_DEIM.cpp index 751ff2e1b..80190e38b 100644 --- a/unit_tests/test_DEIM.cpp +++ b/unit_tests/test_DEIM.cpp @@ -146,7 +146,9 @@ int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); MPI_Init(&argc, &argv); - return RUN_ALL_TESTS(); + int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; } #else // #ifndef CAROM_HAS_GTEST int main() diff --git a/unit_tests/test_GNAT.cpp b/unit_tests/test_GNAT.cpp index d1524b352..e3116511c 100644 --- a/unit_tests/test_GNAT.cpp +++ b/unit_tests/test_GNAT.cpp @@ -151,7 +151,9 @@ int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); MPI_Init(&argc, &argv); - return RUN_ALL_TESTS(); + int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; } #else // #ifndef CAROM_HAS_GTEST int main() diff --git a/unit_tests/test_QDEIM.cpp b/unit_tests/test_QDEIM.cpp index 9e099855a..a231ec5da 100644 --- a/unit_tests/test_QDEIM.cpp +++ b/unit_tests/test_QDEIM.cpp @@ -149,7 +149,9 @@ int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); MPI_Init(&argc, &argv); - return RUN_ALL_TESTS(); + int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; } #else // #ifndef CAROM_HAS_GTEST int main()