.gitlab-ci.yml

# docker image that is used for all ci jobs; see https://hub.docker.com/ for a list of valid docker images
# Image with the centos 7 compilers
# image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/extended
# image: iffregistry.fz-juelich.de/docker-images/centos8-intel-compilers:latest
# image: iffregistry.fz-juelich.de/docker-images/centos8-intel-compilers:latest-ipython3
image: iffregistry.fz-juelich.de/docker-images/centos8-intel-compilers:latest-nvidia_hpc
# list with commands that are executed before each job; every job starts with a fresh docker container

.setup_ifort: &setup_ifort
  - echo "Setting up Intel Fortran compiler"
  # Setting up variables
  - set +e && source compilervars.sh intel64 && set -e
  # Checking location of ifort
  - which ifort
  # Checking ifort version
  - ifort --version
  # Checking location of mpif90
  - which mpiifort
  # Checking mpiifort version
  - mpiifort -v

.setup_gfortran: &setup_gfortran
  - echo "Setting up GNU Fortran compiler"
  # Setting up GNU variables
  - source /opt/rh/gcc-toolset-11/enable
  # Setting up variables
  - set +e && source compilervars.sh intel64 && set -e
  # Setting up MPI variables
  - export PATH=/usr/lib64/mpich/bin:${PATH}
  # Checking location of gfortran
  - which gfortran
  # Checking gfortran version
  - gfortran --version
  # Checking location of mpif90
  - which mpif90
  # Checking mpif90 version
  - mpif90 -v

# pipeline stages; a stage is only run if the previous stage is completed. Each stage contains user-defined jobs
# that are run in parallel
stages:
  - build
  - selfcon:normal
  - selfcon:supercond
  - results
  - plots
  - deploy
  # - test_runtime
  # - deploy:doc
#
# a build job (see `stage` key); jobs can have arbitrary names
#===================================== BUILDS =====================================
build:gnu:normal:
  stage: build
  script:
    - *setup_gfortran
    - cd build
    - FC=mpif90 cmake ../
    - make
  artifacts:
    paths:
      - bin/
    expire_in: 1 day

build:gnu:debug:
  stage: build
  script:
    - *setup_gfortran
    - cd build
    - FC=mpif90 cmake ../ -DDEBUG=ON
    - make 2> >(tee make.warnings)
    - if [ -s make.warnings ] ; then echo "Compiled with warnings"; cat make.warnings ; exit 42 ; fi
  artifacts:
    paths:
      - make.warnings
      - bin/
    expire_in: 1 day
  allow_failure:
    exit_codes: 42

build:intel:debug:
  stage: build
  script:
    - *setup_ifort
    - cd build
    - cmake ../ -DDEBUG=ON
    - make 2> >(tee make.warnings)
    - if [ -s make.warnings ] ; then echo "Compiled with warnings"; cat make.warnings ; exit 42 ; fi
  artifacts:
    paths:
      - make.warnings
      - bin/
    expire_in: 1 day
  allow_failure:
    exit_codes: 42

build:intel:normal:
  stage: build
  script:
    - *setup_ifort
    - cd build
    - cmake ../
    - make
  artifacts:
    paths:
      - bin/
    expire_in: 1 day

#===================================== TESTS =====================================
# after build job(s) are completed, tests can be run; artifacts from all previous stages are transferred automatically

# ** Anchors **
# Going to TEST_DIR, setting calculation type and running calculation
.run_test: &run_test
  # Printing Python version
  - python --version
  # Going to current test folder
  - cd ${TEST_DIR}
  # Setting up
  - sed -i "s/-> itype = .*/-> itype = ${ITYPE}/g" input
  # Setting up 4 omp threads
  - export OMP_NUM_THREADS=8
  # Running the code with 2 MPI ranks (TODO: mpi needs to be tested to return to original values)
  - mpirun -np 1 ${TITAN_DIR}/bin/titan_intel.exe
  # Printing any error file, if existing
  - if [ -f error* ] ; then grep -H '' error* ; fi
  # Printing runtime
  - grep "Finished on" output/*

# Create scResults from output file
.create_scResults_file: &create_scResults_file
  - sed -n '/Self-consistent ground state/,/\(.*iterations \)/p' output/* | head -n -1 | cut -c-77 > results/scResult

# Excluding files if they are not supposed to be checked and
# Comparing all remaining files in results folder with the ones in ./results_correct, using tolerance ${TOL}
.compare_files: &compare_files |
  if [ ! -z ${EXCLUDE} ]
  then
  find ./results -name *${EXCLUDE}* -exec rm -rf {} +
  fi
  for file in `find results/ -type f`
  do
    echo "Found file ${file} in results/"
    [[ -z ${TOL} ]] && echo "Comparing file $(basename ${file}) with default tolerance `grep -m 1 "tol =" ${TITAN_DIR}/scripts/compare_output.py | awk -F= '{print $2}'`" || echo "Comparing file $(basename ${file}) with tolerance ${TOL}"
    python ${TITAN_DIR}/scripts/compare_output.py ${file} ${file/results/results_correct} ${TOL}
  done

.keep_results: &keep_results
  artifacts:
    paths:
      - ${TEST_DIR}/results/
    when: always
    expire_in: 1 day

# ** Jobs **
#============ Normal Self-consistency ============#
Nb_100_Monolayer:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:normal
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Nb_100_Monolayer
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Ni_bulk:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:normal
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Ni_bulk_timeprop
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Fe_bulk:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:normal
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Fe_bulk_susc
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Fe_transv_constr:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:normal
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../../..
    TEST_DIR: examples/Fe_dimer_bconstr/transv
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Fe_full_constr:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:normal
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../../..
    TEST_DIR: examples/Fe_dimer_bconstr/full
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

graphene_pz:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:normal
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/graphene
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Co_monolayer_paoflow:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:normal
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Co_monolayer_paoflow
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

#============ Superconductor Self-consistency ============#
Nb_100_Superconducting_Monolayer:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: selfcon:supercond
  needs: ["build:intel:normal","Nb_100_Monolayer"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Nb_100_Superconducting_Monolayer
    ITYPE: 1
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *create_scResults_file
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

#============ Other results ============#
Nb_100_Monolayer:ldos:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","Nb_100_Monolayer"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Nb_100_Monolayer
    ITYPE: 2
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Nb_100_Monolayer:bands:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","Nb_100_Monolayer"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Nb_100_Monolayer
    ITYPE: 4
    EXCLUDE: weights
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

graphene_pz:ldos:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","graphene_pz"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/graphene
    ITYPE: 2
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

graphene_pz:bands:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","graphene_pz"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/graphene
    ITYPE: 4
    EXCLUDE: weights
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

graphene_pz:isoenergy:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","graphene_pz"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/graphene
    ITYPE: 5
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Fe_bulk:susc:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","Fe_bulk"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Fe_bulk_susc
    ITYPE: 7
    EXCLUDE: Slope
    TOL: 1e-5
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Ni_bulk:timeprop:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","Ni_bulk"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Ni_bulk_timeprop
    ITYPE: 11
    EXCLUDE: checkpoint
    TOL: 1e-6
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

Co_monolayer_paoflow:bands_with_ldos:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","Co_monolayer_paoflow"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Co_monolayer_paoflow
    EXCLUDE: weights
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    # Calculating LDOS
    - export ITYPE=2
    - *run_test
    # Returning to initial folder
    - cd ${TITAN_DIR}
    # Calculating Band structure
    - export ITYPE=4
    - *run_test
    - *compare_files
  # Keep results always
  <<: *keep_results

Co_monolayer_paoflow:mei:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal","Co_monolayer_paoflow"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/Co_monolayer_paoflow
    # Calculating Magnetic Exchange Interaction tensor
    ITYPE: 6
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    # Decreasing number of points
    - sed -i "s/-> nQvec.*/-> nQvec = 5/g" ${TEST_DIR}/input
    - *run_test
    - *compare_files
  # Keep results always
  <<: *keep_results

nanoribbon:responses:
  image: iffregistry.fz-juelich.de/docker-images/centos7-intel-compilers/slurm-daemon:latest
  stage: results
  needs: ["build:intel:normal"]
  variables:
    TITAN_DIR: ../..
    TEST_DIR: examples/nanoribbon
    ITYPE: 8
    EXCLUDE: checkpoint
    TOL: 1e-7
  script:
    # Setting up Intel Fortran Compiler
    - *setup_ifort
    - *run_test
    - *compare_files
  # Keep results on failure (to be able to check)
  <<: *keep_results

#============ Plots ============#
Nb_100_Monolayer:ldos:plot:
  image: iffregistry.fz-juelich.de/docker-images/titan-images
  stage: plots
  needs: ["Nb_100_Monolayer:ldos"]
  tags:
    - intel
  variables:
    # Setting up executables folder
    TITAN_DIR: ../..
    # Setting up example folder
    TEST_DIR: examples/Nb_100_Monolayer
  script:
    # Checking python version
    - python --version
    # Going to current test folder
    - cd ${TEST_DIR}
    # Creating new plot
    - python ${TITAN_DIR}/scripts/plot_ldos.py results/TSOC/1Sites/LDOS/ldosu_site=1_norb=9_nkpt=1024_eta=1.0E-03_ev.dat results/TSOC/1Sites/LDOS/ldosd_site=1_norb=9_nkpt=1024_eta=1.0E-03_ev.dat --xlim="[-0.5,1.5]" --centeref --output="./results/ldos.png"
    # Comparing with existing (correct) plot
    - python ${TITAN_DIR}/scripts/compare_images.py ./results/ldos.png ./results_correct/ldos.png
  # Keep results on failure (to be able to check)
  <<: *keep_results


Co_monolayer_paoflow:bands_with_ldos:plot:
  image: iffregistry.fz-juelich.de/docker-images/titan-images
  stage: plots
  needs: ["Co_monolayer_paoflow:bands_with_ldos"]
  tags:
    - intel
  variables:
    # Setting up executables folder
    TITAN_DIR: ../..
    # Setting up example folder
    TEST_DIR: examples/Co_monolayer_paoflow
  script:
    # Checking python version
    - python --version
    # Going to current test folder
    - cd ${TEST_DIR}
    # Creating new plot
    - python ${TITAN_DIR}/scripts/plot_bands_with_ldos.py results/TSOC/1Sites/BS/bandstructure_kdir=GXMG_norb=9_nkpt=1024_eta=1.0E-03_ev.dat results/TSOC/1Sites/LDOS/ldosu_site=1_norb=9_nkpt=1024_eta=1.0E-03_ev.dat results/TSOC/1Sites/LDOS/ldosd_site=1_norb=9_nkpt=1024_eta=1.0E-03_ev.dat --evlabel --output="./results/bands_with_ldos.png"
    # Comparing with existing (correct) plot
    - python ${TITAN_DIR}/scripts/compare_images.py ./results/bands_with_ldos.png ./results_correct/bands_with_ldos.png
  # Keep results on failure (to be able to check)
  <<: *keep_results


build:page:
  image: iffregistry.fz-juelich.de/docker-images/titan-images
  stage: build
  tags:
    - intel
  script:
    - pip install sphinx
    - pip install sphinx-rtd-theme
    - pip install sphinx-tabs
    - pip install sphinx-design
    - pip install sphinx-last-updated-by-git
    - cd doc/
    - make html
  artifacts:
    paths:
      - doc/_build/html

pages:
  image: iffregistry.fz-juelich.de/docker-images/titan-images
  stage: deploy
  needs:
    - build:page
  tags:
    - intel
  script:
    - mv doc/_build/html/ public/
  artifacts:
    paths:
      - public
  # only:
  #   - master