From 4e6ad2134dc303093beed827767908371f99f8a6 Mon Sep 17 00:00:00 2001 From: Rafal Kapuscik Date: Thu, 16 Apr 2020 11:31:52 +0200 Subject: [PATCH 1/2] Add scripts for kokoro Signed-off-by: Rafal Kapuscik --- .github/kokoro/common.sh | 32 ++++++++++++++++ .github/kokoro/steps/git.sh | 29 +++++++++++++++ .github/kokoro/steps/hostinfo.sh | 30 +++++++++++++++ .github/kokoro/steps/hostsetup.sh | 62 +++++++++++++++++++++++++++++++ .github/kokoro/test.sh | 29 +++++++++++++++ 5 files changed, 182 insertions(+) create mode 100644 .github/kokoro/common.sh create mode 100644 .github/kokoro/steps/git.sh create mode 100644 .github/kokoro/steps/hostinfo.sh create mode 100644 .github/kokoro/steps/hostsetup.sh create mode 100644 .github/kokoro/test.sh diff --git a/.github/kokoro/common.sh b/.github/kokoro/common.sh new file mode 100644 index 00000000..49feedf9 --- /dev/null +++ b/.github/kokoro/common.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -e + +# Check the SCRIPT_XXX values are set. +if [ -z "${SCRIPT_SRC}" ]; then + echo 'Parent script must do SCRIPT_SRC="$(realpath ${BASH_SOURCE[0]})"' + exit 1 +fi +if [ -z "${SCRIPT_DIR}" ]; then + echo 'Parent script must do SCRIPT_DIR="$(dirname "${SCRIPT_SRC}")"' + exit 1 +fi + +# Check the KOKORO environment variables are correct +if [ -z "$KOKORO_DIR" ]; then + echo "$$KOKORO_DIR environment variable is missing ('$KOKORO_DIR')!" + exit 1 +fi + +# Close STDERR FD +exec 2<&- +# Redirect STDERR to STDOUT +exec 2>&1 + +# Get us into the github checkout source directory. +cd github/$KOKORO_DIR/ + +# Run the common setup steps +source ./.github/kokoro/steps/hostsetup.sh +source ./.github/kokoro/steps/hostinfo.sh +source ./.github/kokoro/steps/git.sh diff --git a/.github/kokoro/steps/git.sh b/.github/kokoro/steps/git.sh new file mode 100644 index 00000000..155e7704 --- /dev/null +++ b/.github/kokoro/steps/git.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +echo +echo "========================================" +echo "Git log" +echo "----------------------------------------" +git log -n5 --stat +echo "----------------------------------------" + +echo +echo "========================================" +echo "Git fetching tags" +echo "----------------------------------------" +# Don't fail if there are no tags +git fetch --tags || true +echo "----------------------------------------" + +echo +echo "========================================" +echo "Git version info" +echo "----------------------------------------" +git log -n1 +echo "----------------------------------------" +git describe --tags || true +echo "----------------------------------------" +git describe --tags --always || true +echo "----------------------------------------" diff --git a/.github/kokoro/steps/hostinfo.sh b/.github/kokoro/steps/hostinfo.sh new file mode 100644 index 00000000..84263d36 --- /dev/null +++ b/.github/kokoro/steps/hostinfo.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +set -e + +echo +echo "========================================" +echo "Host Environment" +echo "----------------------------------------" +export +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host CPU" +echo "----------------------------------------" +export CORES=$(nproc --all) +echo "Cores: $CORES" +echo +echo "Memory" +echo "----------------------------------------" +cat /proc/meminfo +echo "----------------------------------------" +export MEM_GB=$(($(awk '/MemTotal/ {print $2}' /proc/meminfo)/(1024*1024))) +echo "Total Memory (GB): $MEM_GB" +export MEM_PER_RUN=16 +export MEM_CORES=$(($MEM_GB/$MEM_PER_RUN)) +echo "Simultaneous runs ($MEM_PER_RUN GB each): $MEM_CORES" +export MAX_CORES_NO_MIN=$(($MEM_CORES>$CORES?$CORES:$MEM_CORES)) +export MAX_CORES=$(($MAX_CORES_NO_MIN<1?1:$MAX_CORES_NO_MIN)) +echo "Max cores: $MAX_CORES" diff --git a/.github/kokoro/steps/hostsetup.sh b/.github/kokoro/steps/hostsetup.sh new file mode 100644 index 00000000..9b649aa9 --- /dev/null +++ b/.github/kokoro/steps/hostsetup.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +set -e + +echo +echo "========================================" +echo "Removing older packages" +echo "----------------------------------------" +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host adding PPAs" +echo "----------------------------------------" +sudo apt-add-repository 'ppa:ubuntu-toolchain-r/test' +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host updating packages" +echo "----------------------------------------" +sudo apt-get update +echo "----------------------------------------" + +echo +echo "========================================" +echo "Host remove packages" +echo "----------------------------------------" +sudo apt-get remove -y \ + python-pytest \ + + +sudo apt-get autoremove -y + +echo "----------------------------------------" +echo +echo "========================================" +echo "Host install packages" +echo "----------------------------------------" +sudo apt-get install -y \ + bash \ + build-essential \ + cmake \ + coreutils \ + git \ + make \ + tclsh \ + +if [ -z "${BUILD_TOOL}" ]; then + export BUILD_TOOL=make +fi + +export CC=gcc-7 +export CXX=g++-7 + +echo "----------------------------------------" + +echo +echo "========================================" +echo "Setting up environment env" +echo "----------------------------------------" +echo "----------------------------------------" diff --git a/.github/kokoro/test.sh b/.github/kokoro/test.sh new file mode 100644 index 00000000..f7f3b491 --- /dev/null +++ b/.github/kokoro/test.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -e + +cd github/$KOKORO_DIR/ + +source ./.github/kokoro/steps/hostsetup.sh +source ./.github/kokoro/steps/hostinfo.sh +source ./.github/kokoro/steps/git.sh + +echo +echo "===========================================" +echo "Building UHDM" +echo "-------------------------------------------" +( + make + sudo make install +) +echo "-------------------------------------------" + +echo +echo "===========================================" +echo "Executing UHDM tests" +echo "-------------------------------------------" +( + make test + make test_install +) +echo "-------------------------------------------" From a84d38a68ef22311e48a30da3cb29fb9cd53ca25 Mon Sep 17 00:00:00 2001 From: Rafal Kapuscik Date: Thu, 16 Apr 2020 11:39:24 +0200 Subject: [PATCH 2/2] Add kokoro config files Signed-off-by: Rafal Kapuscik --- .github/kokoro/continuous-test.cfg | 18 ++++++++++++++ .github/kokoro/kokoro-config.py | 38 ++++++++++++++++++++++++++++++ .github/kokoro/presubmit-test.cfg | 18 ++++++++++++++ 3 files changed, 74 insertions(+) create mode 100644 .github/kokoro/continuous-test.cfg create mode 100644 .github/kokoro/kokoro-config.py create mode 100644 .github/kokoro/presubmit-test.cfg diff --git a/.github/kokoro/continuous-test.cfg b/.github/kokoro/continuous-test.cfg new file mode 100644 index 00000000..eadd2779 --- /dev/null +++ b/.github/kokoro/continuous-test.cfg @@ -0,0 +1,18 @@ +# Format: //devtools/kokoro/config/proto/build.proto +# Generated from .github/kokoro/kokoro-cfg.py +# To regenerate run: +# cd .github/kokoro/ && python3 kokoro-cfg.py +build_file: "UHDM-continuous-test/.github/kokoro/test.sh" +timeout_mins: 4320 +env_vars { + key: "KOKORO_TYPE" + value: "continuous" +} +env_vars { + key: "KOKORO_DIR" + value: "UHDM-continuous-test" +} +env_vars { + key: "CI" + value: "test" +} diff --git a/.github/kokoro/kokoro-config.py b/.github/kokoro/kokoro-config.py new file mode 100644 index 00000000..4460005f --- /dev/null +++ b/.github/kokoro/kokoro-config.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +""" +Generates kokoro config files based on template. +""" + +ci_config = """\ +# Format: //devtools/kokoro/config/proto/build.proto +# Generated from .github/kokoro/kokoro-cfg.py +# To regenerate run: +# cd .github/kokoro/ && python3 kokoro-cfg.py +build_file: "UHDM-%(kokoro_type)s-%(ci)s/.github/kokoro/%(ci)s.sh" +timeout_mins: 4320 +env_vars { + key: "KOKORO_TYPE" + value: "%(kokoro_type)s" +} +env_vars { + key: "KOKORO_DIR" + value: "UHDM-%(kokoro_type)s-%(ci)s" +} +env_vars { + key: "CI" + value: "%(ci)s" +} +""" + +for ci in ['test']: + with open("continuous-%s.cfg" % ci, "w") as f: + f.write(ci_config % { + 'ci': ci, + 'kokoro_type': 'continuous', + }) + + with open("presubmit-%s.cfg" % ci, "w") as f: + f.write(ci_config % { + 'ci': ci, + 'kokoro_type': 'presubmit', + }) diff --git a/.github/kokoro/presubmit-test.cfg b/.github/kokoro/presubmit-test.cfg new file mode 100644 index 00000000..65334e95 --- /dev/null +++ b/.github/kokoro/presubmit-test.cfg @@ -0,0 +1,18 @@ +# Format: //devtools/kokoro/config/proto/build.proto +# Generated from .github/kokoro/kokoro-cfg.py +# To regenerate run: +# cd .github/kokoro/ && python3 kokoro-cfg.py +build_file: "UHDM-presubmit-test/.github/kokoro/test.sh" +timeout_mins: 4320 +env_vars { + key: "KOKORO_TYPE" + value: "presubmit" +} +env_vars { + key: "KOKORO_DIR" + value: "UHDM-presubmit-test" +} +env_vars { + key: "CI" + value: "test" +}