.gitlab-ci.yml

stages:
  - build
  - private_deploy
  - test
  - public_deploy

image: ${CI_DOCKER_REGISTRY}/llvm-build:3

variables:

  # move all these to CI/CD settings
  GCC_REL_NAME: "esp-2022r1-RC1"
  GCC_REL_VER: "gcc11_2_0"
  NEWLIB_REPO: "newlib-cygwin"
  NEWLIB_REF: "esp_based_on_4_1_0"
  BINUTILS_REPO: "binutils-gdb"
  BINUTILS_REF: "esp_based_on_binutils-2_35"
  XTENSA_OVERLAYS_REPO: "xtensa-overlays"
  XTENSA_OVERLAYS_REF: "master"
  # TODO: update vars below to tags names after related branches are merged in those repos
  XTENSA_CLANG_TOOLCHAIN_REF: "release_universal_clang_14.0.0_gcc_8.4.0"
  LLVM_GCC_TESTSUITE_REF: "feature/toolchain_build_script"

  PLATFORM_NAME_LINUX: "linux-amd64"
  PLATFORM_NAME_WIN: "win64"
  PLATFORM_NAME_MACOS: "macos"

  ARCHIVE_TOOL_LINUX: "tar -cJf"
  UNARCHIVE_TOOL_LINUX: "tar -xf"
  ARCHIVE_EXT_LINUX: "tar.xz"

  ARCHIVE_TOOL_WIN: "zip -9 -r"
  UNARCHIVE_TOOL_WIN: "unzip"
  ARCHIVE_EXT_WIN: "zip"

  ARCHIVE_TOOL_MACOS: "tar -cJf"
  UNARCHIVE_TOOL_MACOS: "tar -xf"
  ARCHIVE_EXT_MACOS: "tar.xz"

  DIST_DIR: "dist"
  BUILD_DIR: "_build"
  DOWNLOADS_DIR: "_downloads"

###########################################################################
#################### START OF TEMPORARY LEGACY CODE #######################
# TODO: the code below is to be removed after migration to new build script
  CONF_TARGET: "xtensa-esp32-elf"
  XTENSA_CLANG_TOOLCHAIN: "${CONF_TARGET}-clang"
##################### END OF TEMPORARY LEGACY CODE ########################
###########################################################################

.use_ci_tools: &use_ci_tools |
  curl -sSL ${CIT_LOADER_URL} -o cit_loader.sh && sh cit_loader.sh
  source citools/import_functions

.add_gitlab_key: &add_gitlab_key |
  cit_add_ssh_key "${GITLAB_KEY}"

before_script:
  - *use_ci_tools
  - *add_gitlab_key

# Prepare release name/number
.get_release_name: &get_release_name |
  # using annotated tags
  REL_NUM=$(git describe --abbrev=7)
  REL_SFX="llvm14_0_0"
  REL_NAME=${REL_SFX}-${REL_NUM}-${PLATFORM_NAME}
  ARCHIVE_NAME=${REL_NAME}.${ARCHIVE_EXT}
  echo "PLATFORM_NAME: $PLATFORM_NAME"
  echo "REL_NUM: $REL_NUM"
  echo "REL_NAME: $REL_NAME"
  echo "ARCHIVE_NAME: $ARCHIVE_NAME"

# Get an existing crosstool-ng builds for all chips
.get_gcc_toolchain: &get_gcc_toolchain |
  declare -a XTENSA_CPUS=("esp32"
                         "esp32s2"
                         "esp32s3")
  for ((i = 0; i < ${#XTENSA_CPUS[@]}; i++)); do
    XTENSA_CPU=${XTENSA_CPUS[$i]}
    GCC_TOOLCHAIN_ARCH=xtensa-${XTENSA_CPU}-elf-${GCC_REL_VER}-${GCC_REL_NAME}-${PLATFORM_NAME}.${ARCHIVE_EXT}
    wget --no-verbose https://dl.espressif.com/github_assets/espressif/crosstool-NG/releases/download/${GCC_REL_NAME}/${GCC_TOOLCHAIN_ARCH}
    ${UNARCHIVE_TOOL} ${GCC_TOOLCHAIN_ARCH}
  done;
  GCC_TOOLCHAIN_ARCH=riscv32-esp-elf-${GCC_REL_VER}-${GCC_REL_NAME}-${PLATFORM_NAME}.${ARCHIVE_EXT}
  wget --no-verbose https://dl.espressif.com/github_assets/espressif/crosstool-NG/releases/download/${GCC_REL_NAME}/${GCC_TOOLCHAIN_ARCH}
  ${UNARCHIVE_TOOL} ${GCC_TOOLCHAIN_ARCH}

.get_clang_toolchain_build_scripts: &get_clang_toolchain_build_scripts |
  git clone -b ${XTENSA_CLANG_TOOLCHAIN_REF} ${GITLAB_SSH_SERVER}/${XTENSA_CLANG_TOOLCHAIN_REPO}
  cp -r xtensa-clang-toolchain/* .

# LLVM Build System used the remote address to show detailed version info, we'll change it to the public repository
.fix_origin_remote_for_public: &fix_origin_remote_for_public |
  git remote set-url origin "${GH_REPO_HTTPS}"

# Pack the toolchain
.package_toolchain: &package_toolchain |
  ${ARCHIVE_TOOL} ${ARCHIVE_NAME} esp-clang/
  mkdir -p ${DISTRO_DIR}
  mv ${ARCHIVE_NAME} ${DISTRO_DIR}/
  echo "${ARCHIVE_NAME}" > ${DISTRO_DIR}/file_${PLATFORM_NAME}

.build_template:
  stage: build
  tags: [ "amd64", "build" ]
  allow_failure: true
  artifacts:
    paths:
      - ${DIST_DIR}/
      - ${BUILD_DIR}/clang_tests.log
      - ${BUILD_DIR}/clang_build.log
    when: always
    expire_in: 1 day
  variables:
    BUILD_TOOLCHAIN_CMD_ARGS: ""
    # use separate dist dir for universal toolchain
    # TODO: remove this var after switching to universal toolchain builds
    DIST_DIR: "dist_new"
  script:
    - *get_release_name
    - mkdir ${DOWNLOADS_DIR}
    - pushd ${DOWNLOADS_DIR}
    - export ESP_GCC_TOOLCHAIN_DIST_BASE=$PWD
    - *get_gcc_toolchain
    - git clone -b ${NEWLIB_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${NEWLIB_REPO}.git
    - export NEWLIB_PATH=$PWD/${NEWLIB_REPO}
    - git clone -b ${BINUTILS_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${BINUTILS_REPO}.git
    - export BINUTILS_PATH=$PWD/${BINUTILS_REPO}
    - git clone -b ${XTENSA_OVERLAYS_REF} --single-branch ${GITLAB_SSH_SERVER}/idf/${XTENSA_OVERLAYS_REPO}.git
    - export XTENSA_OVERLAYS_PATH=$PWD/${XTENSA_OVERLAYS_REPO}
    - popd
    - *get_clang_toolchain_build_scripts
    - *fix_origin_remote_for_public
    - export ESP_GCC_TOOLCHAIN_REL_VER=${GCC_REL_NAME}
    - export LLVM_PROJECT_PATH=$PWD
    - export BUILD_PATH=$PWD/${BUILD_DIR}
    - mkdir -p ${BUILD_PATH}
    - export USE_PARALLEL_LINK_JOBS=2
    - ${BUILD_TOOLCHAIN_CMD} --llvm-path=${LLVM_PROJECT_PATH} --newlib-path=${NEWLIB_PATH}
      --gcc-toolchains-path=${ESP_GCC_TOOLCHAIN_DIST_BASE} --binutils-path=${BINUTILS_PATH}
      --xtensa-overlays-path=${XTENSA_OVERLAYS_PATH} ${BUILD_TOOLCHAIN_CMD_ARGS} ${BUILD_PATH} 2>&1 > ${BUILD_PATH}/clang_build.log
    # Run unit tests for native build only.
    # Run as non-root user because permission tests fail when run by root.
    - export BUILD_HOST=$(gcc -dumpmachine)
    - export LLVM_BUILD_PATH=${LLVM_PROJECT_PATH}/llvm/build-Release-${CONF_HOST}
    - if [ "${CONF_HOST}" == "${BUILD_HOST}" ]; then
        echo "Run unit tests for native build";
        useradd -m test_runner;
        chown -R test_runner ${LLVM_BUILD_PATH};
        touch ${BUILD_PATH}/clang_tests.log;
        chmod o+w ${BUILD_PATH}/clang_tests.log;
        runuser -l test_runner -c 'cmake --build '${LLVM_BUILD_PATH}' --target check-all 2>&1 > '${BUILD_PATH}'/clang_tests.log';
      fi
    - export DISTRO_DIR=$PWD/$DIST_DIR
    - pushd ${BUILD_PATH}
    - *package_toolchain
    - popd

build_x86_64-linux-gnu:
  extends: .build_template
  variables:
    CONF_HOST: "x86_64-linux-gnu"
    PLATFORM_NAME: "${PLATFORM_NAME_LINUX}"
    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
    ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
    BUILD_TOOLCHAIN_CMD: "./build-toolchain.sh"

build_x86_64-w64-mingw32:
  extends: .build_template
  needs:
    - job: build_x86_64-linux-gnu
  before_script:
    - *use_ci_tools
    - *add_gitlab_key
    # get ARCHIVE_NAME for Linux release. Modify vars to make get_release_name working properly
    - export PLATFORM_NAME_ORIG=${PLATFORM_NAME}
    - export ARCHIVE_EXT_ORIG=${ARCHIVE_EXT}
    - export PLATFORM_NAME=${PLATFORM_NAME_LINUX}
    - export ARCHIVE_EXT=${ARCHIVE_EXT_LINUX}
    - *get_release_name
    # restore modified vars 
    - export PLATFORM_NAME=${PLATFORM_NAME_ORIG}
    - export ARCHIVE_EXT=${ARCHIVE_EXT_ORIG}
    # unpack Linux release to re-use it as native Clang for Windows build
    - mkdir -p esp-clang-${PLATFORM_NAME_LINUX}
    - ${UNARCHIVE_TOOL_LINUX} ${DIST_DIR}/${ARCHIVE_NAME} -C esp-clang-${PLATFORM_NAME_LINUX}
    # we do not want to keep artifacts from 'x86_64-linux-gnu' job
    - rm -rf ${DIST_DIR}
    - rm -rf ${BUILD_DIR}
    # add build command args speciifc for Windows build
    - export BUILD_TOOLCHAIN_CMD_ARGS="--host=${CONF_HOST} --native-esp-clang-path=$PWD/esp-clang-${PLATFORM_NAME_LINUX}"
  variables:
    CONF_HOST: "x86_64-w64-mingw32"
    PLATFORM_NAME: "${PLATFORM_NAME_WIN}"
    ARCHIVE_TOOL: "${ARCHIVE_TOOL_WIN}"
    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_WIN}"
    ARCHIVE_EXT: "${ARCHIVE_EXT_WIN}"
    BUILD_TOOLCHAIN_CMD: "./build-toolchain-win.sh"

test_x86_64-linux-gnu:
  stage: test
  tags: [ "amd64", "build" ]
  allow_failure: true
  needs:
    - job: build_x86_64-linux-gnu
  variables:
    PLATFORM_NAME: "${PLATFORM_NAME_LINUX}"
    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
    ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
    # use separate dist dir for universal toolchain
    # TODO: remove this var after switching to universal toolchain builds
    DIST_DIR: "dist_new"
  script:
    - *get_release_name
    - ${UNARCHIVE_TOOL} ${DIST_DIR}/${ARCHIVE_NAME}
    # getting testsuite
    - git clone -b ${LLVM_GCC_TESTSUITE_REF} --depth 1 $GITLAB_SSH_SERVER/idf/llvm-xtensa-testsuite.git
    # preparing testsuite
    - export PATH=${PWD}/esp-clang/bin:$PATH
    - cd llvm-xtensa-testsuite
    # qemu
    - ./qemu_esp32_install.sh
    # run testsuite for esp32
    - ./run_esp32_tests.sh

###########################################################################
#################### START OF TEMPORARY LEGACY CODE #######################
# TODO: the code below is to be removed after migration to new build script
.get_release_name_old: &get_release_name_old |
  # using annotated tags
  REL_NUM=$(git describe --abbrev=7)
  REL_SFX="llvm14_0_0"
  REL_NAME=${CONF_TARGET}-${REL_SFX}-${REL_NUM}-${PLATFORM_NAME}
  ARCHIVE_NAME=${REL_NAME}.${ARCHIVE_EXT}
  echo "PLATFORM_NAME: $PLATFORM_NAME"
  echo "REL_NUM: $REL_NUM"
  echo "REL_NAME: $REL_NAME"
  echo "ARCHIVE_NAME: $ARCHIVE_NAME"

.get_gcc_toolchain_old: &get_gcc_toolchain_old |
  wget --no-verbose https://dl.espressif.com/github_assets/espressif/crosstool-NG/releases/download/esp-2021r2-patch3/${XTENSA_GCC_TOOLCHAIN}
  ${UNARCHIVE_TOOL} ${XTENSA_GCC_TOOLCHAIN}
  mv xtensa-esp32-elf ${XTENSA_CLANG_TOOLCHAIN}

.package_toolchain_old: &package_toolchain_old |
  ${ARCHIVE_TOOL} ${ARCHIVE_NAME} ${XTENSA_CLANG_TOOLCHAIN}/
  mkdir -p ${DIST_DIR}
  mv ${ARCHIVE_NAME} ${DIST_DIR}/
  echo "${ARCHIVE_NAME}" > ${DIST_DIR}/file_${PLATFORM_NAME}_${CONF_TARGET}

.build_template_old:
  stage: build
  tags: [ "amd64", "build" ]
  artifacts:
    paths:
      - ${DIST_DIR}/
    when: always
    expire_in: 10 day
  variables:
    XTENSA_CLANG_TOOLCHAIN_REF: "esp-20220415-r14.0.0"
    GCC_REL_NAME: "gcc8_4_0-esp-2021r2-patch3"
  script:
    - *get_release_name_old
    - *get_gcc_toolchain_old
    - *fix_origin_remote_for_public
    - *get_clang_toolchain_build_scripts
    - ${BUILD_TOOLCHAIN_CMD} "${XTENSA_CLANG_TOOLCHAIN}"
    - *package_toolchain_old

linux_amd64_build:
  extends: .build_template_old
  variables:
    PLATFORM_NAME: "${PLATFORM_NAME_LINUX}"
    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
    ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
    XTENSA_GCC_TOOLCHAIN: "xtensa-esp32-elf-${GCC_REL_NAME}-linux-amd64.tar.gz"
    BUILD_TOOLCHAIN_CMD: "./build-toolchain-linux.sh"

win64_build:
  extends: .build_template_old
  variables:
    PLATFORM_NAME: "${PLATFORM_NAME_WIN}"
    ARCHIVE_TOOL: "${ARCHIVE_TOOL_WIN}"
    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_WIN}"
    ARCHIVE_EXT: "${ARCHIVE_EXT_WIN}"
    XTENSA_GCC_TOOLCHAIN: "xtensa-esp32-elf-${GCC_REL_NAME}-win64.zip"
    BUILD_TOOLCHAIN_CMD: "./build-toolchain-win.sh"

macos_amd64_build:
  extends: .build_template_old
  variables:
    PLATFORM_NAME: "${PLATFORM_NAME_MACOS}"
    ARCHIVE_TOOL: "${ARCHIVE_TOOL_MACOS}"
    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_MACOS}"
    ARCHIVE_EXT: "${ARCHIVE_EXT_MACOS}"
    XTENSA_GCC_TOOLCHAIN: "xtensa-esp32-elf-${GCC_REL_NAME}-macos.tar.gz"
    BUILD_TOOLCHAIN_CMD: "./build-toolchain-macos.sh"

linux_amd64_testsuite:
  stage: test
  tags: [ "amd64", "build" ]
  needs:
    - job: linux_amd64_build
  variables:
    PLATFORM_NAME: "${PLATFORM_NAME_LINUX}"
    ARCHIVE_TOOL: "${ARCHIVE_TOOL_LINUX}"
    UNARCHIVE_TOOL: "${UNARCHIVE_TOOL_LINUX}"
    ARCHIVE_EXT: "${ARCHIVE_EXT_LINUX}"
  script:
    - *get_release_name_old
    - ${UNARCHIVE_TOOL} ${DIST_DIR}/${ARCHIVE_NAME}

    # getting testsuite
    - git clone -b ${LLVM_GCC_TESTSUITE_REF} --depth 1 $GITLAB_SSH_SERVER/idf/llvm-xtensa-testsuite.git

    # preparing testsuite
    - export PATH=$PATH:${PWD}/${XTENSA_CLANG_TOOLCHAIN}/bin/
    - cd llvm-xtensa-testsuite

    # qemu
    - ./qemu_esp32_install.sh

    # run testsuite for esp32
    - ./run_esp32_tests.sh

##################### END OF TEMPORARY LEGACY CODE ########################
###########################################################################

upload_to_http:
  stage: private_deploy
  when: manual
  allow_failure: true
  tags: [ "deploy", "shiny" ]
  variables:
    # force the fetch strategy to clean old archives up in dist/ dir
    GIT_STRATEGY: fetch
  before_script:
    - *use_ci_tools
  script:
    - cit_add_ssh_key "${HTTP_UPLOAD_KEY}"
    # List of archives
    - FILES=$(find ${DIST_DIR} -name file_\* -exec cat {} \+)
    - cd ${DIST_DIR}
    - scp ${FILES} ${HTTP_UPLOAD_DIR}/ct-ng/llvm-builds
    # Show info
    - echo -e "\nArchives were published there:\n\n$(for n in ${FILES}; do echo "${HTTP_PUBLIC_DIR}/ct-ng/llvm-builds/${n}"; done)\n"

upload_to_github:
  stage: public_deploy
  when: manual
  allow_failure: true
  only:
    - tags
  tags: [ "amd64", "internet" ]
  image: espressif/github-hub:2
  variables:
    GIT_STRATEGY: fetch
    GITHUB_TOKEN: "${GH_TOKEN}"
    GITHUB_REPO: "${GH_REPO_HTTPS}"
    TAG: "${CI_COMMIT_TAG}"
  before_script: []
  script:
    - ls -l dist*/
    - git remote add github ${GH_REPO_HTTPS}
    - hub release show ${TAG} || { echo "Please create a release on GitHub with ${TAG} tag at first"; exit 1; }
    # List of archives
    - FILES=$(find ${DIST_DIR} -name file_\* -exec cat {} \+)
    - cd ${DIST_DIR}
    - ls -l $FILES
    # Upload archives
    - for n in ${FILES}; do hub release edit -m "" -a "${n}" "${TAG}"; done