From 5f04399d6a6c0cc2d4bb2b101e9a3d93dc968443 Mon Sep 17 00:00:00 2001 From: Cole Kendrick Date: Thu, 19 Sep 2024 07:52:47 -0700 Subject: [PATCH 1/7] Add configuration for gitlab CI --- .gitlab/.gitlab-ci.yml | 53 +++++++++++++++++++++++++++++ .gitlab/run_tests.sh | 70 ++++++++++++++++++++++++++++++++++++++ .gitlab/runners/dane.yml | 55 ++++++++++++++++++++++++++++++ .gitlab/runners/lassen.yml | 30 ++++++++++++++++ 4 files changed, 208 insertions(+) create mode 100644 .gitlab/.gitlab-ci.yml create mode 100755 .gitlab/run_tests.sh create mode 100644 .gitlab/runners/dane.yml create mode 100644 .gitlab/runners/lassen.yml diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml new file mode 100644 index 000000000..c4237ed5d --- /dev/null +++ b/.gitlab/.gitlab-ci.yml @@ -0,0 +1,53 @@ +variables: + CUSTOM_CI_BUILDS_DIR: "/usr/workspace/$$USER/libROM_gitlab_runner" + + GIT_STRATEGY: fetch + GIT_SUBMODULE_STRATEGY: recursive + GIT_DEPTH: 1 + GIT_SUBMODULE_DEPTH: 1 + + DEFAULT_BRANCH: master + + ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} + ALLOC_QUEUE: pci + ALLOC_TIME: 30 + ALLOC_BANK: asccasc + + TEST_SCRIPT: .gitlab/run_tests.sh + + ON_LASSEN: "OFF" + #ON_DANE: "OFF" + +stages: + - allocate + - build + - release + +workflow: + rules: + # skip running branch pipelines if a MR is open for the branch + - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'web' + - if: $CI_COMMIT_TAG + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +include: + # This include is required for LC with Gitlab 17+ + # Refer to https://hpc.llnl.gov/technical-bulletins/bulletin-568 + - project: 'lc-templates/id_tokens' + file: 'id_tokens.yml' + - .gitlab/runners/lassen.yml + - .gitlab/runners/dane.yml + +# Define actual CI jobs here: +dane_gcc_12_1_1: + variables: + MODULE_LIST: cmake gcc/12.1.1 + extends: .job_on_dane + +lassen_gcc_12_2_1: + variables: + MODULE_LIST: cmake/3.23.1 gcc/12.2.1 + extends: .job_on_lassen diff --git a/.gitlab/run_tests.sh b/.gitlab/run_tests.sh new file mode 100755 index 000000000..0e7416104 --- /dev/null +++ b/.gitlab/run_tests.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +modules=${MODULE_LIST:-""} +mpiexec_executable=${MPIEXEC_EXECUTABLE:-"srun"} +# If using flux, append "run" after the flux executable path +if [[ "${mpiexec_executable}" == "flux" ]] +then + mpiexec_executable="$(which ${mpiexec_executable}) run" + flux jobs + flux resource list +else + mpiexec_executable="$(which ${mpiexec_executable})" +fi + +mpiexec_preflags=${MPIEXEC_PREFLAGS:-""} +host=$(hostname) +build_type=${BUILD_TYPE:-"Debug"} +# note: toolchain file here is relative to repo root dir +toolchain_file=${TOOLCHAIN_FILE:-"../cmake/toolchains/default-toss_4_x86_64_ib-librom-dev.cmake"} +use_mfem=${USE_MFEM:-"Off"} +use_gslib=${MFEM_USE_GSLIB:-"Off"} +librom_flags=${LIBROM_FLAGS:-""} + +basehost=${host//[[:digit:]]/} + +echo ${host} + +build_dir=build_${host}_${CI_PIPELINE_ID}_$(date +%F_%H_%M_%S) + +if [[ -n ${modules} ]] +then + module load ${modules} +fi + +# ---- setup googletest ---- +DEPS_DIR="${CI_BUILD_DIR}/${basehost}_deps" +echo $DEPS_DIR +if [[ ! -d "${DEPS_DIR}" ]]; then + mkdir ${DEPS_DIR} && cd ${DEPS_DIR} + git clone https://github.com/google/googletest + cd googletest && mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=./ .. && make && make install + cd ${CI_PROJECT_DIR} +fi +PATH=${DEPS_DIR}/googletest:${DEPS_DIR}/googletest/build:$PATH +echo ${PATH} +# -------------------------- + +#./scripts/compile.sh -d -r + +mkdir ${build_dir} +cd ${build_dir} +pwd + +cmake -DCMAKE_TOOLCHAIN_FILE=${toolchain_file} \ + -DCMAKE_BUILD_TYPE=${build_type} \ + -DMPIEXEC_EXECUTABLE=${mpiexec_executable} \ + -DMPIEXEC_PREFLAGS=${mpiexec_preflags} \ + -DUSE_MFEM=${use_mfem} \ + -DMFEM_USE_GSLIB=${use_gslib} \ + -DENABLE_TESTS=ON \ + -DLIBROM_FLAGS="${librom_flags}" .. + +make -j + +ctest -VV --output-on-failure diff --git a/.gitlab/runners/dane.yml b/.gitlab/runners/dane.yml new file mode 100644 index 000000000..0b06a9de4 --- /dev/null +++ b/.gitlab/runners/dane.yml @@ -0,0 +1,55 @@ +.retry: + retry: + max: 2 + when: + - runner_system_failure + +.on_dane: + extends: + - .retry + tags: + - dane + - shell + rules: + - if: '$ON_DANE == "OFF"' + when: never + - if: $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $DEFAULT_BRANCH + - if: '$CI_JOB_NAME =~ /release_resources_dane/' + when: always + - when: on_success + +allocate_resources_dane: + variables: + GIT_STRATEGY: none + extends: + - .on_dane + stage: allocate + script: + - salloc -N 1 --reservation=ci -A ${ALLOC_BANK} --time=${ALLOC_TIME} --no-shell --job-name=${ALLOC_NAME} + +release_resources_dane: + variables: + GIT_STRATEGY: none + extends: + - .on_dane + stage: release + script: + - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) + - ([[ -n "${JOBID}" ]] && scancel ${JOBID}) + when: always + +.job_on_dane: + extends: .on_dane + stage: build + needs: ["allocate_resources_dane"] + variables: + MPIEXEC_EXECUTABLE: srun + MPIEXEC_PREFLAGS: "--cpu-bind=cores -v" + script: + - echo "JOB NAME ${ALLOC_NAME}" + - export JOBID=$(squeue -h --name=${ALLOC_NAME} --format=%A) + - echo "SLURM ID ${JOBID}" + - srun $( [[ -n "${JOBID}" ]] && echo "--jobid=${JOBID}" ) -N 1 -t ${ALLOC_TIME} -v --overlap ${TEST_SCRIPT} diff --git a/.gitlab/runners/lassen.yml b/.gitlab/runners/lassen.yml new file mode 100644 index 000000000..a92f69dfe --- /dev/null +++ b/.gitlab/runners/lassen.yml @@ -0,0 +1,30 @@ +.retry: + retry: + max: 2 + when: + - runner_system_failure + +.on_lassen: + extends: + - .retry + tags: + - lassen + - shell + rules: + - if: '$ON_LASSEN == "OFF"' + when: never + - if: $CI_PIPELINE_SOURCE == "push" + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_BRANCH == $DEFAULT_BRANCH + - when: on_success + +.job_on_lassen: + extends: .on_lassen + stage: build + needs: [] + variables: + MPIEXEC_EXECUTABLE: jsrun + MPIEXEC_PREFLAGS: "-a 1 -c 1 -g 1" + script: + - bsub -q ${ALLOC_QUEUE} -W ${ALLOC_TIME} -G ${ALLOC_BANK} -J ${ALLOC_NAME} -nnodes 1 -Is ${TEST_SCRIPT} From e0961b9c2e90087d0501e9889b1ffefcab279f4e Mon Sep 17 00:00:00 2001 From: Cole Kendrick Date: Thu, 19 Sep 2024 08:11:46 -0700 Subject: [PATCH 2/7] Update source events --- .gitlab/.gitlab-ci.yml | 2 +- .gitlab/runners/dane.yml | 2 +- .gitlab/runners/lassen.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index c4237ed5d..cfbd307aa 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -28,7 +28,7 @@ workflow: # skip running branch pipelines if a MR is open for the branch - if: $CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS && $CI_PIPELINE_SOURCE == "push" when: never - - if: $CI_PIPELINE_SOURCE == 'merge_request_event' + - if: $CI_PIPELINE_SOURCE == 'external_pull_request_event' - if: $CI_PIPELINE_SOURCE == 'web' - if: $CI_COMMIT_TAG - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/.gitlab/runners/dane.yml b/.gitlab/runners/dane.yml index 0b06a9de4..e0d2889b1 100644 --- a/.gitlab/runners/dane.yml +++ b/.gitlab/runners/dane.yml @@ -15,7 +15,7 @@ when: never - if: $CI_PIPELINE_SOURCE == "push" when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - if: $CI_COMMIT_BRANCH == $DEFAULT_BRANCH - if: '$CI_JOB_NAME =~ /release_resources_dane/' when: always diff --git a/.gitlab/runners/lassen.yml b/.gitlab/runners/lassen.yml index a92f69dfe..beedab917 100644 --- a/.gitlab/runners/lassen.yml +++ b/.gitlab/runners/lassen.yml @@ -15,7 +15,7 @@ when: never - if: $CI_PIPELINE_SOURCE == "push" when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - if: $CI_COMMIT_BRANCH == $DEFAULT_BRANCH - when: on_success From 05358facaf737ee1865615070c30a75710949603 Mon Sep 17 00:00:00 2001 From: Cole Kendrick Date: Thu, 19 Sep 2024 09:30:30 -0700 Subject: [PATCH 3/7] Fix build directory --- .gitlab/run_tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab/run_tests.sh b/.gitlab/run_tests.sh index 0e7416104..dcb0d40ad 100755 --- a/.gitlab/run_tests.sh +++ b/.gitlab/run_tests.sh @@ -37,7 +37,7 @@ then fi # ---- setup googletest ---- -DEPS_DIR="${CI_BUILD_DIR}/${basehost}_deps" +DEPS_DIR="${CI_BUILDS_DIR}/${basehost}_deps" echo $DEPS_DIR if [[ ! -d "${DEPS_DIR}" ]]; then mkdir ${DEPS_DIR} && cd ${DEPS_DIR} @@ -59,7 +59,7 @@ pwd cmake -DCMAKE_TOOLCHAIN_FILE=${toolchain_file} \ -DCMAKE_BUILD_TYPE=${build_type} \ -DMPIEXEC_EXECUTABLE=${mpiexec_executable} \ - -DMPIEXEC_PREFLAGS=${mpiexec_preflags} \ + -DMPIEXEC_PREFLAGS="${mpiexec_preflags}" \ -DUSE_MFEM=${use_mfem} \ -DMFEM_USE_GSLIB=${use_gslib} \ -DENABLE_TESTS=ON \ From 6d8e2cf15146c59881e9166aeaccb62d92860b55 Mon Sep 17 00:00:00 2001 From: Cole Kendrick Date: Tue, 24 Sep 2024 10:05:21 -0700 Subject: [PATCH 4/7] Update serial unit test command for MPI --- unit_tests/CMakeLists.txt | 4 +++- unit_tests/test_DEIM.cpp | 4 +++- unit_tests/test_GNAT.cpp | 4 +++- unit_tests/test_QDEIM.cpp | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/unit_tests/CMakeLists.txt b/unit_tests/CMakeLists.txt index 3532b126b..51829801c 100644 --- a/unit_tests/CMakeLists.txt +++ b/unit_tests/CMakeLists.txt @@ -60,7 +60,9 @@ foreach(stem IN LISTS unit_test_stems) target_compile_features(test_${stem} PRIVATE cxx_std_11) target_compile_definitions(test_${stem} PRIVATE CAROM_HAS_GTEST) - add_test(NAME test_${stem} COMMAND test_${stem}) + # serial tests + set(test_command sh -c "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 ${MPIEXEC_PREFLAGS} test_${stem}") + add_test(NAME test_${stem} COMMAND ${test_command}) endforeach(stem) # Add parallel tests diff --git a/unit_tests/test_DEIM.cpp b/unit_tests/test_DEIM.cpp index 751ff2e1b..80190e38b 100644 --- a/unit_tests/test_DEIM.cpp +++ b/unit_tests/test_DEIM.cpp @@ -146,7 +146,9 @@ int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); MPI_Init(&argc, &argv); - return RUN_ALL_TESTS(); + int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; } #else // #ifndef CAROM_HAS_GTEST int main() diff --git a/unit_tests/test_GNAT.cpp b/unit_tests/test_GNAT.cpp index d1524b352..e3116511c 100644 --- a/unit_tests/test_GNAT.cpp +++ b/unit_tests/test_GNAT.cpp @@ -151,7 +151,9 @@ int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); MPI_Init(&argc, &argv); - return RUN_ALL_TESTS(); + int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; } #else // #ifndef CAROM_HAS_GTEST int main() diff --git a/unit_tests/test_QDEIM.cpp b/unit_tests/test_QDEIM.cpp index 9e099855a..a231ec5da 100644 --- a/unit_tests/test_QDEIM.cpp +++ b/unit_tests/test_QDEIM.cpp @@ -149,7 +149,9 @@ int main(int argc, char* argv[]) { ::testing::InitGoogleTest(&argc, argv); MPI_Init(&argc, &argv); - return RUN_ALL_TESTS(); + int result = RUN_ALL_TESTS(); + MPI_Finalize(); + return result; } #else // #ifndef CAROM_HAS_GTEST int main() From 8e761f3ffde305ea217e0587ff07c922fc80018b Mon Sep 17 00:00:00 2001 From: Cole Kendrick Date: Thu, 26 Sep 2024 06:21:32 -0700 Subject: [PATCH 5/7] Add initial reg test job --- .gitlab/.gitlab-ci.yml | 17 +++++++++++++++++ .gitlab/runners/dane.yml | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index cfbd307aa..6e46da50f 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -51,3 +51,20 @@ lassen_gcc_12_2_1: variables: MODULE_LIST: cmake/3.23.1 gcc/12.2.1 extends: .job_on_lassen + +regtest_dane_gcc_12_1_1: + variables: + MODULE_LIST: cmake gcc/12.1.1 + LLNL_SLURM_SCHEDULER_PARAMETERS: --res=ci + MPIEXEC_EXECUTABLE: srun + MPIEXEC_PREFLAGS: "--cpu-bind=cores -v" + stage: build + rules: + - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_COMMIT_BRANCH != $DEFAULT_BRANCH + when: manual + tags: + - batch + extends: .on_dane + script: + - module load ${MODULE_LIST} + - ./regression_tests/run_regression_tests.sh diff --git a/.gitlab/runners/dane.yml b/.gitlab/runners/dane.yml index e0d2889b1..39c65959c 100644 --- a/.gitlab/runners/dane.yml +++ b/.gitlab/runners/dane.yml @@ -16,7 +16,7 @@ - if: $CI_PIPELINE_SOURCE == "push" when: never - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - - if: $CI_COMMIT_BRANCH == $DEFAULT_BRANCH + - if: $CI_COMMIT_BRANCH != $DEFAULT_BRANCH - if: '$CI_JOB_NAME =~ /release_resources_dane/' when: always - when: on_success From c2949b6568f4a068563a91d4dc38b98284095883 Mon Sep 17 00:00:00 2001 From: Cole Kendrick Date: Thu, 26 Sep 2024 06:42:22 -0700 Subject: [PATCH 6/7] Run test pipelines on push --- .gitlab/.gitlab-ci.yml | 6 ++++-- .gitlab/runners/dane.yml | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 6e46da50f..f9744a2ca 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: ALLOC_NAME: ${CI_PROJECT_NAME}_ci_${CI_PIPELINE_ID} ALLOC_QUEUE: pci - ALLOC_TIME: 30 + ALLOC_TIME: 15 ALLOC_BANK: asccasc TEST_SCRIPT: .gitlab/run_tests.sh @@ -58,10 +58,12 @@ regtest_dane_gcc_12_1_1: LLNL_SLURM_SCHEDULER_PARAMETERS: --res=ci MPIEXEC_EXECUTABLE: srun MPIEXEC_PREFLAGS: "--cpu-bind=cores -v" - stage: build + stage: allocate rules: - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_COMMIT_BRANCH != $DEFAULT_BRANCH when: manual + - if: $CI_PIPELINE_SOURCE == "push" + when: never tags: - batch extends: .on_dane diff --git a/.gitlab/runners/dane.yml b/.gitlab/runners/dane.yml index 39c65959c..4b42371cb 100644 --- a/.gitlab/runners/dane.yml +++ b/.gitlab/runners/dane.yml @@ -14,7 +14,6 @@ - if: '$ON_DANE == "OFF"' when: never - if: $CI_PIPELINE_SOURCE == "push" - when: never - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" - if: $CI_COMMIT_BRANCH != $DEFAULT_BRANCH - if: '$CI_JOB_NAME =~ /release_resources_dane/' From 70611b799490d3d8efd112415f00d1e11c5f2f8d Mon Sep 17 00:00:00 2001 From: Cole Kendrick Date: Thu, 26 Sep 2024 08:32:34 -0700 Subject: [PATCH 7/7] Update rule for regtest --- .gitlab/.gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index f9744a2ca..b0ff1fb18 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -60,10 +60,10 @@ regtest_dane_gcc_12_1_1: MPIEXEC_PREFLAGS: "--cpu-bind=cores -v" stage: allocate rules: - - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_COMMIT_BRANCH != $DEFAULT_BRANCH - when: manual - - if: $CI_PIPELINE_SOURCE == "push" - when: never + #- if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_COMMIT_BRANCH != $DEFAULT_BRANCH + # when: manual + - when: manual + - allow_failure: true tags: - batch extends: .on_dane