.cirrus.yml

# Installs lit, clones the git submodules, builds LDC and the test
# runners and runs the tests.
# Requires env variables CI_ARCH, CI_OS, EXTRA_CMAKE_FLAGS and PARALLELISM.
common_steps_template: &COMMON_STEPS_TEMPLATE
  install_lit_script: |
    # Install lit
    python3 -m pip install --user setuptools wheel
    # lit v0.11.1 is the latest version to work with Python v3.5 (from Ubuntu 16.04)
    python3 -m pip install --user lit==0.11.1
    python3 -c "import lit.main; lit.main.main();" --version . | head -n 1
  clone_submodules_script: |
    cd $CIRRUS_WORKING_DIR
    git submodule update --init --depth $CIRRUS_CLONE_DEPTH
  build_script: |
    # Build LDC & LDC D unittests & defaultlib unittest runners
    cmake --version
    ninja --version
    cd $CIRRUS_WORKING_DIR/..
    installDir=$PWD/install
    mkdir build
    cd build
    IFS=$'\n' extraFlags=( $(xargs -n1 <<<"$EXTRA_CMAKE_FLAGS") )
    cmake -G Ninja $CIRRUS_WORKING_DIR \
      -DCMAKE_BUILD_TYPE=Release \
      -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
      -DCMAKE_INSTALL_PREFIX=$installDir \
      -DINCLUDE_INSTALL_DIR=$installDir/import \
      -DLDC_LINK_MANUALLY=OFF \
      "${extraFlags[@]}"
    ninja -j$PARALLELISM all ldc2-unittest all-test-runners
    bin/ldc2 -version
  always:
    run_compiler_unittests_script: |
      cd $CIRRUS_WORKING_DIR/../build
      ctest --output-on-failure -R ldc2-unittest
    run_lit_testsuite_script: |
      cd $CIRRUS_WORKING_DIR/../build/tests
      if [[ "$CI_OS-$CI_ARCH" == "linux-aarch64" ]]; then
        # FIXME: the following tests fail
        rm $CIRRUS_WORKING_DIR/tests/debuginfo/nested_gdb.d
        rm $CIRRUS_WORKING_DIR/tests/sanitizers/fuzz_basic.d
        # the following 2 have regressed with LLVM 13
        rm $CIRRUS_WORKING_DIR/tests/sanitizers/msan_{noerror,uninitialized}.d
      elif [[ "$CI_OS-$CI_ARCH" == "osx-arm64" ]]; then
        # FIXME: LeakSanitizer: CHECK failed: sanitizer_allocator_secondary.h:177 "((IsAligned(reinterpret_cast<uptr>(p), page_size_))) != (0)"
        rm $CIRRUS_WORKING_DIR/tests/sanitizers/lsan_memleak.d
      fi
      python3 runlit.py -v -j $PARALLELISM .
    run_dmd_testsuite_script: |
      cd $CIRRUS_WORKING_DIR/../build
      DMD_TESTSUITE_MAKE_ARGS=-j$PARALLELISM ctest -V -R dmd-testsuite
    run_defaultlib_tests_script: |
      # Run defaultlib unittests & druntime integration tests
      cd $CIRRUS_WORKING_DIR/../build
      excludes="dmd-testsuite|ldc2-unittest|lit-tests"
      if [[ "$CI_OS" == "freebsd" ]]; then
        # FIXME: unittest failure for unclear reason
        excludes+='|^std.experimental.allocator-shared$'
      elif [[ "$CI_OS-$CI_ARCH" == "linux-aarch64" ]]; then
        # FIXME: segfaults with enabled optimizations
        excludes+='|^core.thread.fiber(-shared)?$'
        # FIXME: failing unittest(s)
        excludes+='|^std.internal.math.gammafunction'
        # FIXME: failing unittest(s) with enabled optimizations
        excludes+='|^std.math.exponential(-shared)?$'
        # FIXME: failure
        excludes+='|^druntime-test-exceptions-debug$'
      fi
      ctest -j$PARALLELISM --output-on-failure -E "$excludes"

# Performs the extra packaging steps for jobs producing a prebuilt package.
# Requires env variables CI_ARCH, CI_OS, GITHUB_TOKEN and PARALLELISM (and CC for FreeBSD).
packaging_steps_template: &PACKAGING_STEPS_TEMPLATE
  # Install LDC & make portable
  install_script: |
    cd $CIRRUS_WORKING_DIR/..
    cd build
    ninja install > /dev/null
    cd ..
    perl -pi -e s?$PWD/install/?%%ldcbinarypath%%/../?g install/etc/ldc2.conf
    if [[ "$CI_OS" == "freebsd" ]]; then
      perl -pi -e "s?,druntime-ldc\",?,druntime-ldc\", \"-gcc=$CC\",?" install/etc/ldc2.conf
    fi
    cp $CIRRUS_WORKING_DIR/{LICENSE,packaging/README} install
    cat install/etc/ldc2.conf
    # Now rename the installation dir to test portability
    mv install installed
  # Run hello-world integration test with shared libs
  run_shared_libs_integration_test_script: |
    cd $CIRRUS_WORKING_DIR/..
    echo 'void main() { import std.stdio; writefln("Hello world, %d bits", size_t.sizeof * 8); }' > hello.d
    installed/bin/ldc2 hello.d -of=hello -link-defaultlib-shared
    ./hello
  # Run hello-world integration test with LTO
  run_lto_integration_test_script: |
    cd $CIRRUS_WORKING_DIR/..
    installed/bin/ldc2 hello.d -of=hello_thin -flto=thin -defaultlib=phobos2-ldc-lto,druntime-ldc-lto
    ./hello_thin
    installed/bin/ldc2 hello.d -of=hello_full -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto
    ./hello_full
  # Run dynamic-compile integration test
  run_dynamic_compile_integration_test_script: |
    cd $CIRRUS_WORKING_DIR/..
    # FIXME: disabled since LLVM 12, needs https://github.com/ldc-developers/ldc/pull/3184
    if [[ "$CI_OS" == "freebsd" ]]; then
      installed/bin/ldc2 -enable-dynamic-compile -run $CIRRUS_WORKING_DIR/tests/dynamiccompile/array.d
    fi
  # Run ImportC integration test
  run_importC_integration_test_script: |
    cd $CIRRUS_WORKING_DIR/..
    installed/bin/ldc2 -run $CIRRUS_WORKING_DIR/tests/dmd/runnable/test22597.c
  # Build & copy dub
  build_dub_script: |
    cd $CIRRUS_WORKING_DIR/..
    export DMD=$PWD/installed/bin/ldmd2
    git clone --recursive https://github.com/dlang/dub.git
    cd dub
    git checkout "$(cat $CIRRUS_WORKING_DIR/packaging/dub_version)"
    $DMD -run build.d -O -w -linkonce-templates
    cp bin/dub ../installed/bin
    ../installed/bin/dub --version
  # Build & copy dlang tools
  build_dlang_tools_script: |
    cd $CIRRUS_WORKING_DIR/..
    DMD=$PWD/installed/bin/ldmd2
    git clone --recursive https://github.com/dlang/tools.git dlang-tools
    cd dlang-tools
    git checkout "$(cat $CIRRUS_WORKING_DIR/packaging/dlang-tools_version)"
    mkdir bin
    $DMD -w -de -dip1000 rdmd.d -of=bin/rdmd
    $DMD -w -de -dip1000 ddemangle.d -of=bin/ddemangle
    $DMD -w -de -dip1000 DustMite/dustmite.d DustMite/splitter.d DustMite/polyhash.d -of=bin/dustmite
    cp bin/{rdmd,ddemangle,dustmite} ../installed/bin
  # Build & copy reggae
  build_reggae_script: |
    cd $CIRRUS_WORKING_DIR/..
    git clone --recursive https://github.com/atilaneves/reggae.git
    cd reggae
    git checkout "$(cat $CIRRUS_WORKING_DIR/packaging/reggae_version)"
    DFLAGS="-O -linkonce-templates" ../dub/bin/dub build -v --combined --compiler="$PWD/../installed/bin/ldc2"
    cp bin/reggae ../installed/bin
    ../installed/bin/reggae --version -b ninja
  # Pack artifact
  pack_artifact_script: |
    cd $CIRRUS_WORKING_DIR/..
    mkdir artifacts
    if [[ "${CIRRUS_TAG:-}" == v* ]]; then
      artifactID=${CIRRUS_TAG:1}
    else
      artifactID=${CIRRUS_CHANGE_IN_REPO:0:8}
    fi
    artifactName=ldc2-$artifactID-$CI_OS-$CI_ARCH
    mv installed $artifactName
    chmod -R go=rX $artifactName
    tar=tar
    if [[ "$CI_OS" == "freebsd" ]]; then
      tar=gtar
    fi
    $tar -cf - --owner=0 --group=0 $artifactName | 7z a artifacts/$artifactName.tar.xz -si -txz -mx9 -mmt$PARALLELISM
    ls -lh artifacts/ldc2-*.tar.xz
  # Upload to GitHub release (only for commits on the master branch and tags)
  upload_to_github_script: |
    cd $CIRRUS_WORKING_DIR
    if [[ "${CIRRUS_TAG:-}" == v* ]]; then
      tools/upload-to-github.sh $CIRRUS_TAG ../artifacts/ldc2-*.tar.xz
    elif [[ "${CIRRUS_TAG:-}" = "" && "$CIRRUS_PR" = "" && "$CIRRUS_BRANCH" = "master" ]]; then
      tools/upload-to-github.sh CI ../artifacts/ldc2-*.tar.xz
    fi

# Installs Ubuntu 18.04+ prerequisites.
# Requires env variables CI_ARCH, HOST_LDC_VERSION, EXTRA_APT_PACKAGES and EXTRA_CMAKE_FLAGS.
install_ubuntu_prerequisites_template: &INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
  install_prerequisites_script: |
    cd $CIRRUS_WORKING_DIR/..
    nproc
    export DEBIAN_FRONTEND=noninteractive
    if [[ "$EXTRA_CMAKE_FLAGS" = *-DMULTILIB?ON* ]]; then
      dpkg --add-architecture i386
      gcc_pkg="g++-multilib"
      libcurl_pkg="libcurl4 libcurl4:i386"
    else
      gcc_pkg="g++"
      libcurl_pkg="libcurl4"
    fi
    apt-get -q update
    apt-get -yq install \
      git-core cmake ninja-build $gcc_pkg \
      zlib1g-dev $libcurl_pkg curl gdb python3 python3-pip tzdata unzip zip \
      $EXTRA_APT_PACKAGES
    python3 --version
    # Download & extract host LDC
    curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-linux-$CI_ARCH.tar.xz
    mkdir host-ldc
    tar -xf ldc2.tar.xz --strip 1 -C host-ldc
    rm ldc2.tar.xz

# Installs macOS prerequisites.
# Requires env variables CI_ARCH, HOST_LDC_VERSION and LLVM_VERSION.
install_macos_prerequisites_template: &INSTALL_MACOS_PREREQUISITES_TEMPLATE
  install_prerequisites_script: |
    cd $CIRRUS_WORKING_DIR/..
    sysctl -n hw.logicalcpu
    # Download & extract CMake
    curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.20.3/cmake-3.20.3-macos-universal.tar.gz
    mkdir cmake
    tar -xf cmake.tar.gz --strip 3 -C cmake
    rm cmake.tar.gz
    # Download & extract Ninja
    curl -fL --retry 3 --max-time 60 -O https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-mac.zip
    mkdir ninja
    tar -xf ninja-mac.zip -C ninja
    rm ninja-mac.zip
    # Download & extract LDC-flavoured LLVM
    echo 'Using LLVM with enabled assertions'
    assertsSuffix="-withAsserts"
    llvm_tag="CI"
    if [[ "$LLVM_VERSION" == *.* ]]; then llvm_tag="ldc-v$LLVM_VERSION"; fi
    curl -fL --retry 3 --max-time 300 -o llvm.tar.xz https://github.com/ldc-developers/llvm-project/releases/download/$llvm_tag/llvm-$LLVM_VERSION-osx-${CI_ARCH}${assertsSuffix}.tar.xz
    mkdir llvm
    tar -xf llvm.tar.xz --strip 1 -C llvm
    rm llvm.tar.xz
    # Download & extract host LDC
    curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-osx-$CI_ARCH.tar.xz
    mkdir host-ldc
    tar -xf ldc2.tar.xz --strip 1 -C host-ldc
    rm ldc2.tar.xz
    # Install Python3
    brew install python
    python3 --version

environment:
  CIRRUS_CLONE_DEPTH: 50
  HOST_LDC_VERSION: 1.31.0
  LLVM_VERSION: 15.0.7
  GITHUB_TOKEN: ENCRYPTED[0955bd48c8d4e5391446fc0149d0719ad0b63df27ec9e6c180a5730a5b10dc7f28f09d1383423db158d21380ee2b022a]

task:
  name: Ubuntu 18.04 x64 multilib rtSanitizers
  container:
    image: ubuntu:18.04
    cpu: 8
    memory: 16G
  timeout_in: 20m
  environment:
    CI_ARCH: x86_64
    CI_OS: linux
    EXTRA_APT_PACKAGES: "llvm-9-dev libclang-common-9-dev"
    EXTRA_CMAKE_FLAGS: "-DMULTILIB=ON -DRT_SUPPORT_SANITIZERS=ON -DBUILD_LTO_LIBS=ON"
    PARALLELISM: 8
  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
  << : *COMMON_STEPS_TEMPLATE

task:
  name: Ubuntu rolling x64 shared-libs-only gdmd
  # allow failures - gdb v10 came with regressions
  allow_failures: true
  container:
    image: ubuntu:rolling
    cpu: 8
    memory: 16G
  timeout_in: 15m
  environment:
    CI_ARCH: x86_64
    CI_OS: linux
    EXTRA_APT_PACKAGES: "gdmd llvm-dev libclang-common-15-dev"
    EXTRA_CMAKE_FLAGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_LTO_LIBS=ON -DD_COMPILER=gdmd -DLDC_LINK_MANUALLY=ON"
    PARALLELISM: 8
    # for gdmd:
    LANG: C.UTF-8
  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
  << : *COMMON_STEPS_TEMPLATE

task:
  name: Ubuntu 18.04 x64 bootstrap
  container:
    image: ubuntu:18.04
    cpu: 8
    memory: 16G
  timeout_in: 15m
  environment:
    CI_ARCH: x86_64
    CI_OS: linux
    HOST_LDC_VERSION: 1.9.0
    EXTRA_APT_PACKAGES: "llvm-9-dev libclang-common-9-dev"
    EXTRA_CMAKE_FLAGS: "-DBUILD_LTO_LIBS=ON"
    PARALLELISM: 8
  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
  << : *COMMON_STEPS_TEMPLATE

task:
  name: macOS 12 $TASK_NAME_SUFFIX
  allow_failures: true # FIXME
  macos_instance:
    image: ghcr.io/cirruslabs/macos-monterey-xcode:latest
  timeout_in: 25m
  environment:
    CI_OS: osx
    # OS is preset to `darwin`
    OS: osx
    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${CIRRUS_WORKING_DIR}/../ninja:${CIRRUS_WORKING_DIR}/../llvm/bin:${PATH}
    PARALLELISM: 4
    matrix:
      #- TASK_NAME_SUFFIX: x64 (M1)
      #  EXTRA_CMAKE_FLAGS: -DCMAKE_OSX_ARCHITECTURES=x86_64 -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON
      #  CI_ARCH: x86_64
      #- TASK_NAME_SUFFIX: x64 (M1) shared-libs-only
      #  EXTRA_CMAKE_FLAGS: -DCMAKE_OSX_ARCHITECTURES=x86_64 -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON -DBUILD_SHARED_LIBS=ON
      #  CI_ARCH: x86_64
      - TASK_NAME_SUFFIX: arm64
        EXTRA_CMAKE_FLAGS: -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON
        CI_ARCH: arm64
      - TASK_NAME_SUFFIX: arm64 shared-libs-only
        EXTRA_CMAKE_FLAGS: -DD_COMPILER_FLAGS=-gcc=/usr/bin/c++ -DBUILD_LTO_LIBS=ON -DBUILD_SHARED_LIBS=ON
        CI_ARCH: arm64
  brew_cache:
    folder: "$HOME/Library/Caches/Homebrew"
  << : *INSTALL_MACOS_PREREQUISITES_TEMPLATE
  << : *COMMON_STEPS_TEMPLATE

task:
  name: Ubuntu 18.04 aarch64
  arm_container:
    image: ubuntu:18.04
    cpu: 4
    memory: 8G
  timeout_in: 60m
  environment:
    CI_ARCH: aarch64
    CI_OS: linux
    EXTRA_APT_PACKAGES: libxml2-dev p7zip-full
    EXTRA_CMAKE_FLAGS: >-
      -DBUILD_LTO_LIBS=ON
      -DD_COMPILER_FLAGS="-O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto"
      -DEXTRA_CXXFLAGS=-flto=full
      -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++
      -DJITRT_EXTRA_LDFLAGS=-static-libstdc++
      -DLDC_INSTALL_LLVM_RUNTIME_LIBS_OS=aarch64-unknown-linux-gnu
      -DLLVM_ROOT_DIR=$CIRRUS_WORKING_DIR/../llvm
      -DD_COMPILER=$CIRRUS_WORKING_DIR/../bootstrap-ldc/bin/ldmd2
    PARALLELISM: 4
    CLANG_VERSION: '14.0.6' # 15.0.6 requires libtinfo.so.6 (but Ubuntu 18 has v5 only)
    CC: $CIRRUS_WORKING_DIR/../clang/bin/clang
    CXX: $CIRRUS_WORKING_DIR/../clang/bin/clang++
  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
  # Download & extract LDC-flavoured LLVM
  download_prebuilt_llvm_script: |
    cd $CIRRUS_WORKING_DIR/..
    assertsSuffix=""
    if [[ "${CIRRUS_TAG:-}" == "" ]]; then
      echo 'Using LLVM with enabled assertions'
      assertsSuffix="-withAsserts"
    fi
    llvm_tag="CI"
    if [[ "$LLVM_VERSION" = *.* ]]; then llvm_tag="ldc-v$LLVM_VERSION"; fi
    llvm_url="https://github.com/ldc-developers/llvm-project/releases/download/$llvm_tag/llvm-$LLVM_VERSION-linux-$CI_ARCH$assertsSuffix.tar.xz"
    curl -fL --retry 3 --max-time 300 -o llvm.tar.xz $llvm_url
    mkdir -p llvm
    tar -xf llvm.tar.xz --strip 1 -C llvm
    rm llvm.tar.xz
    # Make lld the default linker (likely with enabled assertions unfortunately)
    ln -sf "$PWD/llvm/bin/ld.lld" /usr/bin/ld
    ld --version
  # Download & extract clang
  download_prebuilt_clang_script: |
    cd $CIRRUS_WORKING_DIR/..
    curl -fL --retry 3 --max-time 300 -o clang.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-$CLANG_VERSION/clang+llvm-$CLANG_VERSION-$CI_ARCH-linux-gnu.tar.xz
    mkdir clang
    tar -xf clang.tar.xz --strip 1 -C clang
    rm clang.tar.xz
  clone_submodules_early_script: |
    cd $CIRRUS_WORKING_DIR
    git submodule update --init --depth $CIRRUS_CLONE_DEPTH
  # Build a first LDC as host compiler for the actual build
  build_bootstrap_ldc_script: |
    cd $CIRRUS_WORKING_DIR/..
    mkdir bootstrap-ldc
    cd bootstrap-ldc
    cmake -G Ninja $CIRRUS_WORKING_DIR \
      -DCMAKE_BUILD_TYPE=Release \
      -DLLVM_ROOT_DIR=$PWD/../llvm \
      -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
      -DBUILD_SHARED_LIBS=OFF \
      -DBUILD_LTO_LIBS=ON
    ninja -j$PARALLELISM
    bin/ldc2 -version
  << : *COMMON_STEPS_TEMPLATE
  << : *PACKAGING_STEPS_TEMPLATE

task:
  name: FreeBSD 12.2 x64
  freebsd_instance:
    image_family: freebsd-12-2
    cpu: 4
    memory: 8G
  timeout_in: 60m
  environment:
    CI_ARCH: x86_64
    CI_OS: freebsd
    EXTRA_CMAKE_FLAGS: >-
      -DBUILD_LTO_LIBS=ON
      -DD_COMPILER_FLAGS="-O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto"
      -DEXTRA_CXXFLAGS=-flto=full
    PARALLELISM: 4
    # use clang from the `llvm` ports package to be installed, incl. lld support for .deplibs (`pragma(lib, "execinfo")`)
    CC: clang90
  install_prerequisites_script: |
    cd $CIRRUS_WORKING_DIR/..
    sysctl -n hw.ncpu
    pkg install -y git cmake ninja gmake llvm bash gtar 7-zip
    python3 --version
    python3 -m ensurepip
    # Download & extract host LDC
    curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-freebsd-$CI_ARCH.tar.xz
    mkdir pre-ldc
    gtar -xf ldc2.tar.xz --strip 1 -C pre-ldc
  clone_submodules_early_script: |
    cd $CIRRUS_WORKING_DIR
    git submodule update --init --depth $CIRRUS_CLONE_DEPTH
  # Build a first LDC as host compiler for the actual build
  build_bootstrap_ldc_script: |
    cd $CIRRUS_WORKING_DIR/..
    mkdir host-ldc
    cd host-ldc
    cmake -G Ninja $CIRRUS_WORKING_DIR \
      -DCMAKE_BUILD_TYPE=Release \
      -DD_COMPILER=$PWD/../pre-ldc/bin/ldmd2 \
      -DBUILD_SHARED_LIBS=OFF \
      -DBUILD_LTO_LIBS=ON
    ninja -j$PARALLELISM
    bin/ldc2 -version
  << : *COMMON_STEPS_TEMPLATE
  << : *PACKAGING_STEPS_TEMPLATE