diff --git a/.github/workflows/build-containers-pr.yml b/.github/workflows/build-containers-pr.yml deleted file mode 100644 index c5cca9badd6..00000000000 --- a/.github/workflows/build-containers-pr.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: Docker smoketest build -on: - pull_request: - branches: - - "master" - - "candidate-*.x" - - "!candidate-8.2.*" - - "!candidate-8.0.*" - - "!candidate-7.12.*" - - "!candidate-7.10.*" - - "!candidate-7.8.*" - - "!candidate-7.6.*" - - "!candidate-7.4.*" - - "!candidate-7.2.*" - - "!candidate-7.0.*" - - "!candidate-6.*" - -jobs: - check-skip: - # continue-on-error: true # Uncomment once integration is finished - runs-on: ubuntu-20.04 - # Map a step output to a job output - outputs: - changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }} - steps: - - id: skip_check - uses: hpcc-systems/github-actions/changed-modules@main - with: - github_token: ${{ github.token }} - - build-images: - needs: check-skip - if: ${{ needs.check-skip.outputs.changed }} - runs-on: ubuntu-20.04 - steps: - - name: vars - id: vars - run: | - # echo ::set-output name=container_registry::ghcr.io - # echo ::set-output name=cr_user::${{ github.repository_owner }} - echo ::set-output name=container_registry::docker.io - echo ::set-output name=cr_user::hpccbuilds - echo ::set-output name=build_prbase_sha::${{ github.event.pull_request.base.sha }} - echo ::set-output name=build_prbase_label::${{ github.base_ref }} - echo ::set-output name=build_user::${{ github.actor }} - echo ::set-output name=build_type::RelWithDebInfo - echo ::set-output name=use_cppunit::1 - echo ::set-output name=platform_build::smoketest-platform-build-vcpkg - - - name: tracing - run: | - echo "Base ref = ${{ github.ref }}" - echo "Action = ${{ github.action }}" - echo "Event = ${{ github.event_name }}" - echo "Actor = ${{ github.actor }}" - echo "Ref = ${{ github.ref }}" - echo "base sha = ${{ github.event.pull_request.base.sha }}" - echo "Sha = ${{ github.sha }}" - echo "github.repository = ${{ github.repository }}" - echo "repository_owner = ${{ github.repository_owner }}" - echo "github.workspace = ${{ github.workspace }}" - echo "runner.workspace = ${{ runner.workspace }}" - echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}" - echo "build_prbase_label = ${{ steps.vars.outputs.build_prbase_label }}" - echo "platform_build = ${{ steps.vars.outputs.platform_build }}" - - - name: Checkout PR - uses: actions/checkout@v3 - - - name: Calculate vcpkg vars - id: vcpkg_vars - run: | - echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT - - - name: tracing (vcpkg_vars) - run: | - echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}" - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - with: - driver: docker - - - name: Check if PR-Base prebuilt - id: check-images - run: | - prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }} > /dev/null ; echo $?) - echo prbase_missing=${prbase_missing} - if [[ "${prbase_missing}" -eq 1 ]] - then - echo "Current PR target branch image cannot be found, using latest" - prbase_missing=$(docker manifest inspect ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest > /dev/null ; echo $?) - if [[ "${prbase_missing}" -eq 1 ]] - then - echo "Cannot find the 'latest' target branch image" - echo ::set-output name=prbase_missing::${prbase_missing} - fi - echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-latest - else - echo ::set-output name=platform_prbase_ver::${{ steps.vars.outputs.build_prbase_label }}-${{ github.event.pull_request.base.sha }} - fi - - # Normal expectation is that the following step would normally be skipped - # NB: This is only for the case where the target branch image doesn't already exist. - # The build-containers-target-branch.yml action would normally have built/published this image, - # or they'll be a "latest" which will have been picked up by the check-images step. - - name: branch image - if: ${{ steps.check-images.outputs.prbase_missing == '1' }} - uses: docker/build-push-action@v2 - with: - context: . - file: ./dockerfiles/platform-build/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - tags: | - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-${{ steps.vars.outputs.build_prbase_sha }} - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_prbase_label }}-latest - build-args: | - CR_REPO=${{ steps.vars.outputs.container_registry }} - BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }} - BUILD_USER=${{ github.repository_owner }} - GITHUB_ACTOR=${{ github.actor }} - BUILD_TAG=${{ steps.vars.outputs.build_prbase_sha }} - BUILD_TYPE=${{ steps.vars.outputs.build_type }} - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }} - BUILD_THREADS=${{ steps.vars.outputs.build_threads }} - - - name: PR image - uses: docker/build-push-action@v2 - with: - context: . - file: ./dockerfiles/platform-build-incremental-container/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - build-args: | - CR_USER=${{ steps.vars.outputs.cr_user }} - CR_REPO=${{ steps.vars.outputs.container_registry }} - PLATFORM_PRBASE_VER=${{ steps.check-images.outputs.platform_prbase_ver }} - GITHUB_REPO=${{ github.repository }} - GITHUB_PRREF=${{ github.ref }} - BUILD_THREADS=${{ steps.vars.outputs.build_threads }} diff --git a/.github/workflows/build-containers-target-branch.yml b/.github/workflows/build-containers-target-branch.yml deleted file mode 100644 index dafe221a23e..00000000000 --- a/.github/workflows/build-containers-target-branch.yml +++ /dev/null @@ -1,112 +0,0 @@ -name: Docker target branch build -on: - push: - branches: - - "master" - - "candidate-*.x" - - "!candidate-8.2.*" - - "!candidate-8.0.*" - - "!candidate-7.12.*" - - "!candidate-7.10.*" - - "!candidate-7.8.*" - - "!candidate-7.6.*" - - "!candidate-7.4.*" - - "!candidate-7.2.*" - - "!candidate-7.0.*" - - "!candidate-6.*" - -jobs: - check-skip: - # continue-on-error: true # Uncomment once integration is finished - runs-on: ubuntu-20.04 - # Map a step output to a job output - outputs: - changed: ${{ steps.skip_check.outputs.dockerfiles || steps.skip_check.outputs.platform }} - steps: - - id: skip_check - uses: hpcc-systems/github-actions/changed-modules@main - with: - github_token: ${{ github.token }} - - build-images: - needs: check-skip - if: ${{ needs.check-skip.outputs.changed }} - runs-on: ubuntu-20.04 - steps: - - name: vars - id: vars - run: | - # echo ::set-output name=container_registry::ghcr.io - # echo ::set-output name=cr_user::${{ github.repository_owner }} - echo ::set-output name=container_registry::docker.io - echo ::set-output name=cr_user::hpccbuilds - echo ::set-output name=build_base_sha::${{ github.sha }} - echo ::set-output name=build_base_label::${GITHUB_REF##*/} - echo ::set-output name=build_user::${{ github.actor }} - echo ::set-output name=build_type::RelWithDebInfo - echo ::set-output name=use_cppunit::1 - echo ::set-output name=platform_build::smoketest-platform-build-vcpkg - - - name: tracing - run: | - echo "Action = ${{ github.action }}" - echo "Event = ${{ github.event_name }}" - echo "Actor = ${{ github.actor }}" - echo "Ref = ${{ github.ref }}" - echo "Sha = ${{ github.sha }}" - echo "github.repository = ${{ github.repository }}" - echo "repository_owner = ${{ github.repository_owner }}" - echo "github.workspace = ${{ github.workspace }}" - echo "runner.workspace = ${{ runner.workspace }}" - echo "build_base_sha = ${{ steps.vars.outputs.build_base_sha }}" - echo "build_base_label = ${{ steps.vars.outputs.build_base_label }}" - - - name: Checkout - uses: actions/checkout@v2 - - - name: Calculate vcpkg vars - id: vcpkg_vars - run: | - echo "base_ver=$(git submodule status vcpkg | cut -c2-9)" >> $GITHUB_OUTPUT - - - name: tracing (vcpkg_vars) - run: | - echo "vcpkg_vars.base_ver = ${{ steps.vcpkg_vars.outputs.base_ver }}" - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - # - name: Login to GitHub Container Registry - # uses: docker/login-action@v1 - # with: - # registry: ghcr.io - # username: ${{ github.repository_owner }} - # password: ${{ secrets.GITHUB_TOKEN }} - - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - # build branch image - - name: branch image - uses: docker/build-push-action@v2 - with: - context: . - file: ./dockerfiles/platform-build/Dockerfile - builder: ${{ steps.buildx.outputs.name }} - tags: | - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-${{ steps.vars.outputs.build_base_sha }} - ${{ steps.vars.outputs.container_registry }}/${{ steps.vars.outputs.cr_user }}/${{ steps.vars.outputs.platform_build }}:${{ steps.vars.outputs.build_base_label }}-latest - push: true - build-args: | - CR_REPO=${{ steps.vars.outputs.container_registry }} - BASE_VER=${{ steps.vcpkg_vars.outputs.base_ver }} - BUILD_USER=${{ github.repository_owner }} - BUILD_TAG=${{ steps.vars.outputs.build_base_sha }} - BUILD_TYPE=${{ steps.vars.outputs.build_type }} - GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - USE_CPPUNIT=${{ steps.vars.outputs.use_cppunit }} - BUILD_THREADS=${{ steps.vars.outputs.build_threads }} diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 56f318be832..e7bfab4a02f 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -30,7 +30,7 @@ on: default: false strip-files: type: boolean - description: 'Single Package' + description: 'Strip Debug Symbols' required: false default: true cmake-configuration: @@ -138,7 +138,7 @@ jobs: --mount source="${{ github.workspace }}/build",target=/hpcc-dev/build,type=bind,consistency=delegated \ --mount source="${{ github.workspace }}/.ccache",target=/root/.ccache,type=bind,consistency=delegated \ ${{ steps.vars.outputs.docker_tag }} "\ - cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} && \ + cmake -G Ninja -S /hpcc-dev/${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B /hpcc-dev/build -DHPCC_SOURCE_DIR=/hpcc-dev/HPCC-Platform -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} && \ cmake --build /hpcc-dev/build --parallel ${{ inputs.upload-package == true && '--target package' || ''}}" done diff --git a/.github/workflows/build-gh_runner.yml b/.github/workflows/build-gh_runner.yml index 58e0dbc3052..48823c004c5 100644 --- a/.github/workflows/build-gh_runner.yml +++ b/.github/workflows/build-gh_runner.yml @@ -34,7 +34,7 @@ on: default: false strip-files: type: boolean - description: 'Single Package' + description: 'Strip Debug Symbols' required: false default: true cmake-configuration: @@ -204,8 +204,14 @@ jobs: run: | mkdir -p ${{ github.workspace }}/LN mkdir -p ${{ github.workspace }}/build - cmake ${{ !contains(inputs.os, 'windows') && '-G Ninja' || '' }} -S ./${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} - cmake --build ./build ${{ contains(inputs.os, 'windows') && '--config Release' || ''}} --parallel ${{ inputs.upload-package == true && '--target package' || ''}} + declare -a plugins + plugins=(${{ inputs.single-package == true && '"PLATFORM"' || '"PLATFORM" "CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "PARQUETEMBED" "REDIS" "REMBED" "SQLITE3EMBED" "SQS"' }}) + for plugin in "${plugins[@]}"; do + rm -f ./build/CMakeCache.txt + rm -rf ./build/CMakeFiles + cmake ${{ !contains(inputs.os, 'windows') && '-G Ninja' || '' }} -S ./${{ inputs.ln == true && 'LN' || 'HPCC-Platform' }} -B ./build -DCMAKE_BUILD_TYPE=${{ inputs.build-type }} -DCONTAINERIZED=${{ inputs.containerized == true && 'ON' || 'OFF' }} -DCPACK_STRIP_FILES=${{ inputs.strip-files == true && 'ON' || 'OFF' }} ${{ inputs.single-package == true && '-DINCLUDE_PLUGINS=ON' || '-D$plugin=ON' }} ${{ inputs.cmake-configuration }} ${{ inputs.cmake-configuration-ex }} + cmake --build ./build ${{ contains(inputs.os, 'windows') && '--config RelWithDebInfo' || ''}} --parallel ${{ inputs.upload-package == true && '--target package' || ''}} + done - name: Upload Package if: ${{ inputs.upload-package == true }} diff --git a/.github/workflows/build-test-eclwatch.yml b/.github/workflows/build-test-eclwatch.yml index 13eee0bfa2a..520fea47ba2 100644 --- a/.github/workflows/build-test-eclwatch.yml +++ b/.github/workflows/build-test-eclwatch.yml @@ -38,13 +38,6 @@ jobs: - uses: actions/setup-node@v2 with: node-version: ${{ matrix.node }} - - name: Cache node modules - uses: actions/cache@v2 - with: - path: ./esp/src/node_modules - key: npm-deps-${{ hashFiles('./esp/src/package-lock.json') }} - restore-keys: | - npm-deps-${{ hashFiles('./esp/src/package-lock.json') }} - name: Install Dependencies working-directory: ./esp/src run: npm ci diff --git a/.github/workflows/build-vcpkg.yml b/.github/workflows/build-vcpkg.yml index 7c79c2d72a6..676b7f82b3a 100644 --- a/.github/workflows/build-vcpkg.yml +++ b/.github/workflows/build-vcpkg.yml @@ -1,11 +1,5 @@ name: Test Build -env: - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read" - VCPKG_NUGET_REPOSITORY: "https://github.com/hpcc-systems/vcpkg" - OS_DEPENDENCIES: "bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk - r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake" - on: workflow_dispatch: inputs: @@ -51,6 +45,7 @@ on: - cron: "0 0 * * *" jobs: + build-workflow-dispatch: if: ${{ contains('workflow_dispatch', github.event_name) }} uses: ./.github/workflows/build-docker.yml @@ -75,8 +70,6 @@ jobs: uses: ./.github/workflows/build-docker.yml with: os: ubuntu-23.10 - single-package: true - containerized: false secrets: inherit build-docker-ubuntu-22_04: @@ -84,10 +77,8 @@ jobs: uses: ./.github/workflows/build-docker.yml with: os: ubuntu-22.04 - single-package: true - containerized: false upload-package: true - asset-name: 'docker-package' + asset-name: 'docker-ubuntu-22_04' secrets: inherit test-smoke-docker-ubuntu-22_04: @@ -96,7 +87,7 @@ jobs: uses: ./.github/workflows/test-smoke-gh_runner.yml with: os: ubuntu-22.04 - asset-name: 'docker-package' + asset-name: 'docker-ubuntu-22_04' secrets: inherit test-unit-docker-ubuntu-22_04: @@ -105,7 +96,7 @@ jobs: uses: ./.github/workflows/test-unit-gh_runner.yml with: os: ubuntu-22.04 - asset-name: 'docker-package' + asset-name: 'docker-ubuntu-22_04' secrets: inherit test-ui-docker-ubuntu-22_04: @@ -114,7 +105,7 @@ jobs: uses: ./.github/workflows/test-ui-gh_runner.yml with: os: ubuntu-22.04 - asset-name: 'docker-package' + asset-name: 'docker-ubuntu-22_04' secrets: inherit build-docker-ubuntu-20_04: @@ -122,8 +113,6 @@ jobs: uses: ./.github/workflows/build-docker.yml with: os: ubuntu-20.04 - single-package: true - containerized: false secrets: inherit build-docker-centos-8: @@ -131,8 +120,6 @@ jobs: uses: ./.github/workflows/build-docker.yml with: os: centos-8 - single-package: true - containerized: false secrets: inherit build-docker-centos-7: @@ -140,8 +127,6 @@ jobs: uses: ./.github/workflows/build-docker.yml with: os: centos-7 - single-package: true - containerized: false cmake-configuration-ex: "-DVCPKG_TARGET_TRIPLET=x64-centos-7-dynamic" secrets: inherit @@ -150,56 +135,21 @@ jobs: uses: ./.github/workflows/build-docker.yml with: os: amazonlinux - single-package: true - containerized: false cmake-configuration-ex: "-DVCPKG_TARGET_TRIPLET=x64-amazonlinux-dynamic" secrets: inherit build-gh_runner-ubuntu-22_04: - if: ${{ contains('pull_request,push', github.event_name) }} + if: ${{ contains('schedule,push', github.event_name) }} uses: ./.github/workflows/build-gh_runner.yml with: os: ubuntu-22.04 - single-package: true - containerized: false - upload-package: true - asset-name: 'gh_runner-package' secrets: inherit - test-smoke-gh_runner-ubuntu-22_04: - if: ${{ contains('pull_request,push', github.event_name) }} - needs: build-gh_runner-ubuntu-22_04 - uses: ./.github/workflows/test-smoke-gh_runner.yml - with: - os: ubuntu-22.04 - asset-name: 'gh_runner-package' - secrets: inherit - - test-unit-gh_runner-ubuntu-22_04: - if: ${{ contains('pull_request,push', github.event_name) }} - needs: build-gh_runner-ubuntu-22_04 - uses: ./.github/workflows/test-unit-gh_runner.yml - with: - os: ubuntu-22.04 - asset-name: 'gh_runner-package' - secrets: inherit - - test-ui-gh_runner-ubuntu-22_04: - if: ${{ contains('pull_request,push', github.event_name) }} - needs: build-gh_runner-ubuntu-22_04 - uses: ./.github/workflows/test-ui-gh_runner.yml - with: - os: ubuntu-22.04 - asset-name: 'gh_runner-package' - secrets: inherit - build-gh_runner-ubuntu-20_04: if: ${{ contains('schedule,push', github.event_name) }} uses: ./.github/workflows/build-gh_runner.yml with: os: ubuntu-20.04 - single-package: true - containerized: false secrets: inherit build-gh_runner-windows-2022: @@ -207,9 +157,7 @@ jobs: uses: ./.github/workflows/build-gh_runner.yml with: os: windows-2022 - single-package: true - containerized: false - cmake-configuration: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DINCLUDE_PLUGINS=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' secrets: inherit build-gh_runner-windows-2019: @@ -217,9 +165,7 @@ jobs: uses: ./.github/workflows/build-gh_runner.yml with: os: windows-2019 - single-package: true - containerized: false - cmake-configuration: '-T host=x64 -A x64 -DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + cmake-configuration-ex: '-T host=x64 -A x64 -DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' secrets: inherit build-gh_runner-macos-12: @@ -227,10 +173,8 @@ jobs: uses: ./.github/workflows/build-gh_runner.yml with: os: macos-12 - single-package: true - containerized: false build-type: 'Release' - cmake-configuration: '-DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' secrets: inherit build-gh_runner-macos-11: @@ -238,8 +182,6 @@ jobs: uses: ./.github/workflows/build-gh_runner.yml with: os: macos-11 - single-package: true - containerized: false build-type: 'Release' - cmake-configuration: '-DUSE_OPTIONAL=OFF -DCLIENTTOOLS_ONLY=ON -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' + cmake-configuration-ex: '-DUSE_CPPUNIT=OFF -DCLIENTTOOLS_ONLY=ON -DINCLUDE_PLUGINS=OFF -DUSE_AZURE=OFF -DUSE_CASSANDRA=OFF -DSUPPRESS_CASSANDRAEMBED=ON -DUSE_JAVA=OFF -DUSE_OPENLDAP=OFF' secrets: inherit diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml deleted file mode 100644 index 0428925cbaf..00000000000 --- a/.github/workflows/smoketest.yml +++ /dev/null @@ -1,592 +0,0 @@ -# Smoketest github action -# ======================= -# -# Uses cached instances of previous builds to perform partial (and therefore quicker) incremental builds. -# Once the platform is built and installed (make install), -# it runs the regression suite setup stage within the same github job ('build-and-setup'). -# -# On success, an artifact is built, that contains the install binaries and -# the HPCC binaries that have accumulated from the setup stage. -# -# Once the 'build-and-setup' job is complete, the dependent regression suite jobs and unittest job are launched in parallel. -# The regression suite queries are manually sub-divided into chunks (alphabeticaly), and run via a job matrix for parallism. -# If anything fails, all jobs are aborted (fail-fast=true), and the logs are captured into a published artifact. -# -# NOTES: -# + pre-requisite build dependencies (and runtime dependencies) are listed manually, and must be kept up to date with the -# requirements of the platform. MORE: a list of required build dependencies could be kept within the platform source and picked up -# + 'cacheversion' is purely in case it is necessary to force a cache-miss, i.e. all cached items are based on this version -# + Caching is via github's actions/cache, and is limited to 5GB total per repository, with oldest ejected first, and/or >7 days -# cached builds are tagged with: -# 1) base+ref+SHA (exact match, e.g. hpccbuild-1-7.12.10-6c981c48ae6e35b62d86d8e59e42799c5cd14812) -# 2) base_ref (branch match, e.g. hpccbuild-1-7.12.10) -# 3) base-ref major-minor only (e.g. hpccbuild-1-7.12.) -# 4) base-ref major only (e.g. hpccbuild-1-7.) -# 5) generic cacheversion only (e.g. hpccbuild-1-) -# The cache will initially try to match an exact key match (only true if rerunning PR and SHA same), -# and will then failover to trying to find a match of 2-5, in that order. - - -name: smoketest -env: - cacheversion: 3 - VCPKG_BINARY_SOURCES: "clear;nuget,GitHub,read" - OS_DEPENDENCIES: "bison flex build-essential binutils-dev curl lsb-release libcppunit-dev python3-dev default-jdk - r-base-dev r-cran-rcpp r-cran-rinside r-cran-inline pkg-config libtool autotools-dev automake git cmake" - -on: - pull_request: - branches: - - "master" - - "newbuild" - - "candidate-*" - - "!candidate-7.6.*" - - "!candidate-7.4.*" - - "!candidate-7.2.*" - - "!candidate-7.0.*" - - "!candidate-6.*" - -# NB: this fails to cancel in-flight actions, with error 'Resource not accessible by integration', -# but it's non-fatal. Appears to be dependent on what permissions the PR owner has, i.e. the github token used -# does not have permission to cancel actions. -jobs: - check-skip: - # continue-on-error: true # Uncomment once integration is finished - runs-on: ubuntu-22.04 - # Map a step output to a job output - outputs: - platform: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.platform }} - eclwatch: ${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') || steps.skip_check.outputs.eclwatch }} - - steps: - - id: skip_check - uses: hpcc-systems/github-actions/changed-modules@main - with: - github_token: ${{ github.token }} - - build-and-setup: - name: Build platform and regress setup - needs: check-skip - if: ${{ needs.check-skip.outputs.platform || needs.check-skip.outputs.eclwatch }} - runs-on: ubuntu-22.04 - timeout-minutes: 150 # the build step has its own timeout, however the job can sometimes take time to download dependencies - outputs: - cache-state: ${{ steps.check-cache.outputs.state }} - - steps: - - name: Setup and git clone - id: vars - run: | - if [[ "${{ github.event_name }}" = 'pull_request' ]] - then - echo ::set-output name=branch::${{ github.head_ref }} - echo ::set-output name=base_ref::${{ github.base_ref }} - versioned=$(echo ${{ github.base_ref }} | sed -E -n 's/^(.*)-([0-9]+)\.([0-9]+)\.([0-9]+|x)(-[0-9]+|).*$/\1 \2 \3 \4 \5/p') - if [[ "${versioned}" ]] - then - echo "matched version = ${versioned}" - IFS=' ' read prefix major minor point cand <<< ${versioned} - echo "name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}." - echo "name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}." - echo ::set-output name=base-majorminor-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}.${minor}. - echo ::set-output name=base-majoronly-restore-key::hpccbuild-${{ env.cacheversion }}-${prefix}-${major}. - fi - else - echo ::set-output name=branch::${GITHUB_REF#refs/heads/} - echo ::set-output name=base_ref::${GITHUB_REF#refs/heads/} - fi - if [[ "${{ github.event.pull_request.head.repo.owner.login }}" = "${{ github.repository_owner }}" ]] - then - repository=${{ github.repository }} - else - repository=$(echo -n ${{ github.event.pull_request.head.repo.owner.login }}/ ; echo -n ${{ github.repository }} | sed -E -n 's@^[^/]+/(.+)$@\1@p') - fi - echo ::set-output name=repository::${repository} - echo "Cloning https://github.com/${repository}" - git clone -n https://github.com/${repository} src - if [[ "${{ github.event_name }}" = 'pull_request' ]] - then - cd src - git remote add upstream https://github.com/${{ github.repository }} - git fetch upstream +${{ github.ref }} - fetch_sha=$(git rev-parse --verify FETCH_HEAD) - git fetch upstream ${{ github.base_ref}} - base_ref_sha=$(git rev-parse upstream/${{ github.base_ref }}) - echo ::set-output name=base_ref_sha::${base_ref_sha} - echo ::set-output name=sha::${fetch_sha} - echo "base_ref_sha = ${base_ref_sha}" - echo "fetch_sha = ${fetch_sha}" - else - echo ::set-output name=sha::${{ github.sha }} - echo ::set-output name=base_ref_sha::${{ github.sha }} - fi - - # NB: restore-keys set to e.g. hpccbuild-2-7.12.10, hpccbuild-2-7.12, hpccbuild-2-7, hpccbuild-2- - - name: Fetch cache - id: cache - uses: actions/cache@v2 - with: - path: | - build-cache - merge-patch - src-cache - key: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}-${{ steps.vars.outputs.sha }} - restore-keys: | - hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }} - ${{ steps.vars.outputs.base-majorminor-restore-key }} - ${{ steps.vars.outputs.base-majoronly-restore-key }} - hpccbuild-${{ env.cacheversion }}- - - - name: tracing - run: | - echo "Branch = ${{ steps.vars.outputs.branch }}" - echo "Base ref = ${{ steps.vars.outputs.base_ref }}" - echo "SHA = ${{ steps.vars.outputs.sha}}" - echo "Action = ${{ github.action }}" - echo "Event = ${{ github.event_name }}" - echo "Actor = ${{ github.actor }}" - echo "Ref = ${{ github.ref }}" - echo "Sha = ${{ github.sha }}" - echo "github.repository = ${{ github.repository }}" - echo "repository = ${{ steps.vars.outputs.repository }}" - echo "repository_owner = ${{ github.repository_owner }}" - echo "github.workspace = ${{ github.workspace }}" - echo "runner.workspace = ${{ runner.workspace }}" - echo "github.event.pull_request.head.repo.owner.login = ${{ github.event.pull_request.head.repo.owner.login }}" - - if [[ "${{ github.event_name }}" = 'pull_request' ]] - then - echo "PR base_ref = ${{ github.base_ref }}" - echo "PR head_ref = ${{ github.head_ref }}" - echo "PR base SHA = ${{ github.event.pull_request.base.sha }}" - fi - echo "restore-key1: hpccbuild-${{ env.cacheversion }}-${{ steps.vars.outputs.base_ref }}" - echo "restore-key2: ${{ steps.vars.outputs.base-majorminor-restore-key }}" - echo "restore-key3: ${{ steps.vars.outputs.base-majoronly-restore-key }}" - echo "restore-key4: hpccbuild-${{ env.cacheversion }}-" - - # NB: actions/cache does not set cache-hit to true if restore-keys used. - # So here we: - # 1) check if a cache was restored - # 2) check that the SHA stored in the cache can be found in git - # 3) mv cache into place (build-cache->build) - # 4) checkout source @ base_ref - # 5) get submodules - # 6) apply git merge-patch from cache, bringing source up-to-date with the source used in the cached binary build - # 7) copy src-cache files into place - # 8) touch all files so that 'old' [before original make timestamp], i.e. so behind objects in cache - # 9) checkout pr branch - ensuring only pr files that differ are updated (and hence will rebuild) - # 10) list updated files (for tracing only) - # 11) set 'state' output variable to 'miss' if not absolute match, - # and set to 'hit' if cache is usable. i.e. if set at all, run build and tests. - - name: Check cache - id: check-cache - if: ${{ (steps.cache.outputs.cache-hit != 'true') || contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }} - run: | - if [[ ("${{ steps.cache.outputs.cache-hit }}" == 'true') && ("${{ contains(github.event.pull_request.labels.*.name, 'smoketest-force-rerun') }}" == 'true') ]] - then - echo "Exact cache match, but 'smoketest-force-rerun' label forced rerun" - fi - echo ::set-output name=state::miss - if [[ "${{ contains(github.event.pull_request.labels.*.name, 'smoketest-ignore-cache') }}" == 'true' ]] - then - echo "Cache found, but ignored because 'smoketest-ignore-cache' label set" - rm -rf build-cache merge-patch src-cache - else - if [[ -d build-cache ]] - then - echo "Attempting to use existing cache" - stat build-cache merge-patch src-cache - du -sh build-cache merge-patch src-cache - cd src - if [[ "commit" == "$(git cat-file -t $(cat ../build-cache/cache-sha))" ]] - then - mv ../build-cache ../build - echo "Checking out cache SHA (cached github.base_ref SHA): $(cat ../build/cache-sha)" - git checkout $(cat ../build/cache-sha) - git submodule update --init --recursive --jobs 4 - echo "Applying merge patch" - git apply ../merge-patch - # Add all mods that came from merge-patch into a commit, - # so that the final git checkout will not clash with working tree files - git add -A - git -c user.name='Smoketest' -c user.email='smoke@test.com' commit -m 'merge-patch' - cd ../src-cache - echo "Cached source tree files:" - find . -type f - echo "================" - find . -type f | cpio -p -dum ../src - cd ../src - find . -type f -exec touch -r ../build/cache-timestamp {} + - git checkout ${{ steps.vars.outputs.sha }} - git submodule update --init --recursive --jobs 4 - echo "Changed files (from SHA: $(cat ../build/cache-sha)):" - find -name '.git*' -prune -o -newer ../build/cache-timestamp -type f -print - echo ::set-output name=state::hit - else - echo "Cached SHA $(cat ../build-cache/cache-sha) could not be found). Skipping cache." - fi - fi - fi - - - name: Dependencies22.04 - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed - run: | - sudo apt-get update - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y xmlstarlet - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: Cache miss prep - if: ${{ steps.check-cache.outputs.state == 'miss' }} - run: | - rm -rf build build-cache merge-patch src-cache - mkdir build - mkdir src-cache - cd src - git checkout ${{ steps.vars.outputs.sha }} - git submodule update --init --recursive --jobs 4 - - - name: "Setup NuGet credentials" - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed - run: | - mono `vcpkg fetch nuget | tail -n 1` \ - sources add \ - -source "https://nuget.pkg.github.com/hpcc-systems/index.json" \ - -storepasswordincleartext \ - -name "GitHub" \ - -username "${{ github.actor }}" \ - -password "${{ secrets.GITHUB_TOKEN }}" - - - name: "Remove builtin vcpkg" - if: ${{ github.event_name == matrix.event_name && needs.check-skip.outputs.platform }} - shell: "bash" - run: | - sudo rm -rf "$VCPKG_INSTALLATION_ROOT" - - - name: "vcpkg Bootstrap" - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set then step needed - working-directory: ./src/vcpkg - run: | - ./bootstrap-vcpkg.sh - - # Subsequent cache hit builds use cache-timestamp to ensure all sources are marked 'old', - # except those changed between the cache SHA and the SHA being built. - # NB: BUILD_TAG is overridden to a constant "smoketest", this is to prevent the auto-generated build tag being - # used, which would cause it to change per PR (per SHA), and because it's in a common header, cause a lot of undesirable rebuilding. - - name: Build - timeout-minutes: 120 # Generous, typical build time from clean is < 60 - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed - run: | - cd build - echo "removing CMakeCache.txt" - rm -f CMakeCache.txt - if [[ ! -f cache-timestamp ]]; then touch cache-timestamp; fi - touch ../cmake-timestamp - `../src/vcpkg/vcpkg fetch cmake | tail -n 1` ../src -Wno-dev -DRUNTIME_USER=$(whoami) -DRUNTIME_GROUP=$(id -gn) -DDESTDIR=$(realpath ..)/install -DINCLUDE_PLUGINS=1 -DCMAKE_BUILD_TYPE=Release -DUSE_PYTHON2=0 -DUSE_PYTHON3=1 -DSUPPRESS_SPARK=1 -DUSE_CPPUNIT=1 -DUSE_CASSANDRA=Off -DCMAKE_CXX_FLAGS="-DBUILD_TAG=\\\"dummytag\\\" -DBUILD_VERSION_MAJOR=1 -DBUILD_VERSION_MINOR=1 -DBUILD_VERSION_POINT=0" -DBUILD_TAG="smoketest" -DSTRIP_RELEASE_SYMBOLS=0 -DCMAKE_CXX_FLAGS="-DTRACE_GLOBAL_GROUP=1" - make -j$(nproc) install - - - name: build-logs-artifact - if: ${{ failure() }} - uses: actions/upload-artifact@v2 - with: - name: build-logs - path: | - build/CMakeCache.txt - build/CMakeFiles/CMakeOutput.log - build/CMakeFiles/CMakeError.log - - # Cache contains: - # - make binaries (build-cache) - # - Any sources written since cache-timestamp within the source dir (src-cache) (e.g. AWS writes a config header) - # - cache-timestamp, timestamp of cache-miss build. All files except diffs set to this time, ensuring no existing binaries are rebuilt - # - cache-sha, the SHA of the base_ref used to build. On a cache hit, used to find changed files and ensure they are only ones rebuilt. - # - merge-path, the diffs between the base_ref and the PR commit at the time of PR/cache build. - - name: Prepare cache - if: ${{ steps.check-cache.outputs.state != '' }} # if 'state' set to anything, then step needed - run: | - echo ${{ steps.vars.outputs.base_ref_sha }} > build/cache-sha - mv build build-cache - cd src - find -name '.git*' -prune -o -newer ../cmake-timestamp -type f -print | cpio -p -dum ../src-cache - echo "Merge patch files:" - git diff --name-only ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} - git diff --binary ${{ steps.vars.outputs.base_ref_sha }} ${{ steps.vars.outputs.sha }} > ../merge-patch - - # - alter stock ecl-test.json, to point to install dir (using jq) - # - create a clean copy of the install directory (for later reuse) - # - rm hpcc-data from copy (the post-setup hpcc-data will be moved in when done) - - name: Prepare install artifact - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - run: | - # configure ecl-test.json with correct paths - jq --arg rootdir "${{ github.workspace }}" \ - '.Regress.dropzonePath = $rootdir + "/install" + .Regress.dropzonePath | .Regress.setupExtraParams.OriginalTextFilesOsPath = $rootdir + "/install" + .Regress.setupExtraParams.OriginalTextFilesOsPath | .Regress.setupExtraDParams.HPCCBaseDir = $rootdir + "/install" + .Regress.setupExtraDParams.HPCCBaseDir | .Regress.regressionDir = $rootdir + "/regress" | .Regress.maxAttemptCount = "1" | .Regress.logDir = $rootdir + "/regress/log"' \ - install/opt/HPCCSystems/testing/regress/ecl-test.json > ecl-test.json - mv -f ecl-test.json install/opt/HPCCSystems/testing/regress/ecl-test.json - # configure environment.xml to slavesPerNode=2, channelsPerNode=1 - xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 install/etc/HPCCSystems/environment.xml - cp ${{ github.workspace }}/src/.github/workflows/timeoutcmd install/opt/HPCCSystems/bin/ - cp ${{ github.workspace }}/src/.github/workflows/smoketest-preabort.sh install/opt/HPCCSystems/bin/ - mkdir copy - cp -rp install copy/ - rm -rf copy/install/var/lib/HPCCSystems/hpcc-data - rm -rf copy/install/var/lib/HPCCSystems/queries - - - name: Run regression suite setup - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - timeout-minutes: 10 - run: | - export LANG="en_US.UTF-8" - sudo update-locale - source install/opt/HPCCSystems/sbin/hpcc_setenv - install/opt/HPCCSystems/etc/init.d/hpcc-init start - mkdir -p regress - cd install/opt/HPCCSystems/testing/regress - # force regression suite to timeout after 8 minutes, so it captures ZAP report of any inflight hung queries - timeoutcmd $((8 * 60)) \ - ./ecl-test setup --preAbort 'smoketest-preabort.sh' --pq 2 --generateStackTrace - grep Failure: ${{ github.workspace }}/regress/log/setup_*.log - if [[ "$(grep -ohP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/setup_*.log | awk '{s+=$1} END {print s}')" -gt 0 ]]; then exit 1; fi - ${{ github.workspace }}/install/opt/HPCCSystems/etc/init.d/hpcc-init stop - - - name: regression-setup-logs-artifact - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: regress-setup-logs - path: | - install/var/log/HPCCSystems - regress/ - - # - mv regression suite setup created data from hpcc-data and queries into copy of install - # - create tarball of install for artifact uploading - - name: Finalize install artifact - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - run: | - mv install/var/lib/HPCCSystems/hpcc-data copy/install/var/lib/HPCCSystems/ - mv install/var/lib/HPCCSystems/queries copy/install/var/lib/HPCCSystems/ - [[ -d ${{ github.workspace }}/src/esp/src/test-ui ]] && cp -r ${{ github.workspace }}/src/esp/src/test-ui copy/install/ - cd copy - tar --zstd -cf ../install.tgz install - - - name: Upload install artifact - if: ${{ steps.check-cache.outputs.state != '' && github.event_name == 'pull_request' }} - uses: actions/upload-artifact@v2 - with: - name: installsetup-artifact - path: | - install.tgz - - # Matrix of jobs run in parallel once build+setup above completes successfully. - # All use the post-setup install.tgz artifact, that contains binaries and setup state - # Break each engine run into sets for speed - # NB: each regression suite job, runs these steps: - # - installs dependencies (probably too many for runtime) - # - Starts HPCC - # - Runs regression suite with params { engine, match[pattern,exclude] } - # TODO: needs to process results, capture report, to be assembled by workflow when all jobs done - regression-jobs: - needs: build-and-setup - if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.platform }} - timeout-minutes: 60 # each matrix step has its own timeout, however the job can sometimes take time to download dependencies - runs-on: ubuntu-22.04 - strategy: - fail-fast: true - matrix: - engine: ['hthor', 'thor', 'roxie'] - match: - - pattern: '[a-d]*.ecl' - - pattern: '[e-h]*.ecl' - - pattern: '[i-l]*.ecl' - - pattern: '[m-q]*.ecl' - - pattern: '[r-u]*.ecl' - exclude: teststdlibrary.ecl - - pattern: '[v-z]*.ecl' - include: - - engine: 'hthor' - match: - pattern: teststdlibrary.ecl - - engine: 'thor' - match: - pattern: teststdlibrary.ecl - - engine: 'roxie' - match: - pattern: teststdlibrary.ecl - - steps: - - uses: actions/download-artifact@v2 - with: - name: installsetup-artifact - - - name: vars - id: vars - run: | - echo ::set-output name=matrix-setname::$(echo -n ${{ matrix.match.pattern }} | tr -c "[:alnum:]" _) - - - name: Prerequisites - run: | - sudo apt-get update - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y gdb - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: run - timeout-minutes: 30 # generous, each matrix step shouldn't take more than 10 mins - run: | - export LANG="en_US.UTF-8" - sudo update-locale - tar --zstd -xf install.tgz - source install/opt/HPCCSystems/sbin/hpcc_setenv - install/opt/HPCCSystems/etc/init.d/hpcc-init start - cd install/opt/HPCCSystems/testing/regress - - (cd ecl; ls ${{ matrix.match.pattern }}) > matches.tmp - echo queries are: - cat matches.tmp - if [[ -z "${{ matrix.match.exclude }}" ]] - then - queries="$(cat matches.tmp | tr '\n' ' ')" - else - queries="$(cd ecl; ls ${{ matrix.match.exclude }} | grep -v -f - ../matches.tmp | tr '\n' ' ')" - fi - echo queries after exclude: ${queries} - - # force regression suite to timeout after 28 minutes, so it captures ZAP report of any inflight hung queries - timeoutcmd $((28 * 60)) \ - ./ecl-test query --preAbort 'smoketest-preabort.sh' --pq 2 --target ${{ matrix.engine }} --excludeclass python2,embedded-r,embedded-js,3rdpartyservice,mongodb --generateStackTrace ${queries} - grep Failure: ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log - if [[ "$(grep -oP '(?<=^Failure: )[0-9]+$' ${{ github.workspace }}/regress/log/${{ matrix.engine }}.*.log)" -gt 0 ]]; then exit 1; fi - - - name: regression-run-logs-artifact - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: regression-run-logs-${{ matrix.engine }}-${{ steps.vars.outputs.matrix-setname }} - path: | - install/var/log/HPCCSystems - regress/ - if-no-files-found: error - - - # NB: this doesn't really need the post-setup data files included in install.tgz - # but as this is relatively quick and in parallel with others, it probably doesn't matter - unittests: - needs: build-and-setup - if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' }} - timeout-minutes: 30 # the unittests run step has its own timeout, however the job can sometimes take time to download dependencies - runs-on: ubuntu-22.04 - steps: - - uses: actions/download-artifact@v2 - with: - name: installsetup-artifact - - - name: Prerequisites - run: | - sudo apt-get update - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y gdb - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: run - timeout-minutes: 10 # generous, expected time is approx 1 min. - run: | - export LANG="en_US.UTF-8" - sudo update-locale - tar --zstd -xf install.tgz - install/opt/HPCCSystems/bin/unittests - - # NB: this doesn't really need the post-setup data files included in install.tgz - # but as this is relatively quick and in parallel with others, it probably doesn't matter - eclwatch-ui-tests: - needs: build-and-setup - if: ${{ needs.build-and-setup.outputs.cache-state != '' && github.event_name == 'pull_request' && needs.check-skip.outputs.eclwatch }} - timeout-minutes: 30 # the ui-tests run step has its own timeout, however the job can sometimes take time to download dependencies - runs-on: ubuntu-22.04 - steps: - - uses: actions/download-artifact@v2 - with: - name: installsetup-artifact - - - name: Check-ECLWatch-UI-Test-Directory - id: check - run: | - tar --zstd -xf install.tgz - if [[ ! -d install/test-ui/tests ]] - then - echo "ECLWatch UI test-ui/tests directory missing." - else - javaFilesCount=$(find install/test-ui/tests/ -iname '*.java' -type f -print | wc -l ) - echo "Number of test java files is $javaFilesCount" - if [[ ${javaFilesCount} -eq 0 ]] - then - echo "No java files, do nothing." - else - echo ::set-output name=runtests::true - fi - fi - - - name: Prerequisites - if: steps.check.outputs.runtests - run: | - sudo apt-get update - sudo apt-get install -y git wget net-tools - sudo apt-get install -y tzdata unzip xvfb libxi6 - sudo apt-get install -y default-jdk - sudo apt-get install -y ${{ env.OS_DEPENDENCIES }} - sudo apt-get install -y gdb - - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: UI-Test-Prerequisites - if: steps.check.outputs.runtests - run: | - wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - sudo apt-get install -y ./google-chrome-stable_current_amd64.deb - wget https://chromedriver.storage.googleapis.com/2.41/chromedriver_linux64.zip - unzip chromedriver_linux64.zip - sudo mv chromedriver /usr/bin/chromedriver - sudo chown root:root /usr/bin/chromedriver - sudo chmod +x /usr/bin/chromedriver - wget https://selenium-release.storage.googleapis.com/3.141/selenium-server-standalone-3.141.59.jar - wget http://www.java2s.com/Code/JarDownload/testng/testng-6.8.7.jar.zip - unzip testng-6.8.7.jar.zip - - - name: run - timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added - if: steps.check.outputs.runtests - run: | - export LANG="en_US.UTF-8" - sudo update-locale - source install/opt/HPCCSystems/sbin/hpcc_setenv - install/opt/HPCCSystems/etc/init.d/hpcc-init start - export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)" - pushd install/test-ui - ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 - retCode=$? - echo "UI test done" - [[ $retCode -ne 0 ]] && exit 1 - popd - - - name: eclwatch-ui-test-logs-artifact - if: ${{ failure() || cancelled() }} - uses: actions/upload-artifact@v2 - with: - name: ECLWatchUiTest - path: install/test-ui/eclWatchUiTest.log - if-no-files-found: error diff --git a/.github/workflows/test-smoke-gh_runner.yml b/.github/workflows/test-smoke-gh_runner.yml index 2d65c3b9199..b01fdd3ebbc 100644 --- a/.github/workflows/test-smoke-gh_runner.yml +++ b/.github/workflows/test-smoke-gh_runner.yml @@ -76,11 +76,6 @@ jobs: sudo apt-get install -y ${{ inputs.dependencies }} sudo apt-get install -y gdb - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - name: Install Package shell: "bash" run: | @@ -103,7 +98,6 @@ jobs: working-directory: /opt/HPCCSystems/testing/regress shell: "bash" run: | - # todo: run once before matrix /opt/HPCCSystems/bin/timeoutcmd $((8 * 60)) \ ./ecl-test setup --preAbort '/opt/HPCCSystems/bin/smoketest-preabort.sh' --pq 2 --generateStackTrace grep Failure: /home/runner/HPCCSystems-regression/log/setup_*.log @@ -130,8 +124,8 @@ jobs: working-directory: /opt/HPCCSystems/testing/regress shell: "bash" run: | - # force regression suite to timeout after 60 minutes, so it captures ZAP report of any inflight hung queries - /opt/HPCCSystems/bin/timeoutcmd $((60 * 60)) \ + # force regression suite to timeout after 28 minutes, so it captures ZAP report of any inflight hung queries + /opt/HPCCSystems/bin/timeoutcmd $((28 * 60)) \ ./ecl-test query --preAbort '/opt/HPCCSystems/bin/smoketest-preabort.sh' --pq 2 --target ${{ matrix.engine }} --excludeclass python2,embedded-r,embedded-js,3rdpartyservice,mongodb --generateStackTrace ${{ steps.select-tests.outputs.queries }} grep Failure: /home/runner/HPCCSystems-regression/log/${{ matrix.engine }}.*.log if [[ "$(grep -oP '(?<=^Failure: )[0-9]+$' /home/runner/HPCCSystems-regression/log/${{ matrix.engine }}.*.log)" -gt 0 ]]; then exit 1; fi diff --git a/.github/workflows/test-ui-gh_runner.yml b/.github/workflows/test-ui-gh_runner.yml index ec2a6b79235..391ead72bee 100644 --- a/.github/workflows/test-ui-gh_runner.yml +++ b/.github/workflows/test-ui-gh_runner.yml @@ -36,7 +36,7 @@ jobs: name: ${{ inputs.asset-name }}-ui_test-files path: ${{ inputs.asset-name }}-ui_test-files - - name: Check-ECLWatch-UI-Test-Directory + - name: Check ECLWatch UI Test Directory id: check run: | if [[ ! -d ${{ inputs.asset-name }}-ui_test-files ]] @@ -58,18 +58,19 @@ jobs: shell: "bash" run: | sudo apt-get update - sudo apt-get install -y git wget net-tools - sudo apt-get install -y tzdata unzip xvfb libxi6 - sudo apt-get install -y default-jdk - sudo apt-get install -y ${{ inputs.dependencies }} - sudo apt-get install -y gdb + sudo apt-get install -y \ + git \ + wget \ + net-tools \ + tzdata \ + unzip \ + xvfb \ + libxi6 \ + default-jdk \ + gdb \ + ${{ inputs.dependencies }} - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - - name: UI-Test-Prerequisites + - name: Install UI Dependencies if: steps.check.outputs.runtests run: | wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb @@ -99,14 +100,19 @@ jobs: sudo xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 /etc/HPCCSystems/environment.xml - - name: run - timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added - if: steps.check.outputs.runtests + chmod +x ./${{ inputs.asset-name }}-ui_test-files/* + + - name: Start HPCC-Platform + shell: "bash" run: | export LANG="en_US.UTF-8" - update-locale - source /opt/HPCCSystems/sbin/hpcc_setenv + sudo update-locale sudo /etc/init.d/hpcc-init start + + - name: Run Tests + timeout-minutes: 10 # generous, current runtime is ~1min, this should be increased if new tests are added + if: steps.check.outputs.runtests + run: | export CLASSPATH=".:$(realpath selenium-server-standalone-3.141.59.jar):$(realpath testng-6.8.7.jar)" pushd ${{ inputs.asset-name }}-ui_test-files ./run.sh tests http://localhost:8010 > eclWatchUiTest.log 2>&1 diff --git a/.github/workflows/test-unit-gh_runner.yml b/.github/workflows/test-unit-gh_runner.yml index 904187785e8..69928aa0811 100644 --- a/.github/workflows/test-unit-gh_runner.yml +++ b/.github/workflows/test-unit-gh_runner.yml @@ -35,12 +35,6 @@ jobs: name: ${{ inputs.asset-name }} path: ${{ inputs.asset-name }} - - name: Download Support Files - uses: actions/download-artifact@v3 - with: - name: ${{ inputs.asset-name }}-support-files - path: ${{ inputs.asset-name }}-support-files - - name: Install Dependencies shell: "bash" run: | @@ -48,31 +42,26 @@ jobs: sudo apt-get install -y ${{ inputs.dependencies }} sudo apt-get install -y gdb - curl -fsSL https://deb.nodesource.com/setup_16.x | sudo -E bash - - sudo apt-get install -y nodejs - - echo "node.js version = $(node --version), npm version = $(npm --version), release = $(lsb_release -sc)" - - name: Install Package shell: "bash" run: | sudo apt-get install -y -f ./${{ inputs.asset-name }}/*.deb sudo chown -R $USER:$USER /opt/HPCCSystems - sudo xmlstarlet ed -L -u 'Environment/Software/ThorCluster/@slavesPerNode' -v 2 -u 'Environment/Software/ThorCluster/@channelsPerSlave' -v 1 /etc/HPCCSystems/environment.xml - - chmod +x ./${{ inputs.asset-name }}-support-files/* - sudo cp ./${{ inputs.asset-name }}-support-files/* /opt/HPCCSystems/bin - - - name: Start HPCC-Platform + - name: Run Tests + timeout-minutes: 10 # generous, expected time is approx 1 min. shell: "bash" run: | export LANG="en_US.UTF-8" sudo update-locale - sudo /etc/init.d/hpcc-init start + sudo /opt/HPCCSystems/bin/unittests - - name: Run Tests - timeout-minutes: 10 # generous, expected time is approx 1 min. - shell: "bash" - run: | - /opt/HPCCSystems/bin/unittests + - name: unittests-logs-artifact + if: ${{ failure() || cancelled() }} + uses: actions/upload-artifact@v3 + with: + name: test-unit-${{ inputs.asset-name }} + path: | + /var/log/HPCCSystems + /home/runner/HPCCSystems-regression + if-no-files-found: error diff --git a/cmake_modules/options.cmake b/cmake_modules/options.cmake index a744818799d..0d1dbc1f677 100644 --- a/cmake_modules/options.cmake +++ b/cmake_modules/options.cmake @@ -88,7 +88,7 @@ if (NOT USE_JAVA) endif () if ((INCLUDE_PLUGINS AND NOT SUPPRESS_JAVAEMBED) OR INCLUDE_JAVAEMBED) message(WARNING "JAVAEMBED requires USE_JAVA, forcing SUPPRESS_JAVAEMBED ON") - set (SUPRESS_JAVAEMBED ON) + set (SUPPRESS_JAVAEMBED ON) endif () endif () diff --git a/dockerfiles/vcpkg/build.sh b/dockerfiles/vcpkg/build.sh index 2a624617b69..f314770edc2 100755 --- a/dockerfiles/vcpkg/build.sh +++ b/dockerfiles/vcpkg/build.sh @@ -29,9 +29,9 @@ docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON" function doBuild() { - # docker pull "hpccsystems/platform-build-base-$1:$VCPKG_REF" || true - # docker pull "hpccsystems/platform-build-$1:$VCPKG_REF" || true - # docker pull "hpccsystems/platform-build-$1:$GITHUB_BRANCH" || true + docker pull "hpccsystems/platform-build-base-$1:$VCPKG_REF" || true + docker pull "hpccsystems/platform-build-$1:$VCPKG_REF" || true + docker pull "hpccsystems/platform-build-$1:$GITHUB_BRANCH" || true docker build --progress plain --rm -f "$SCRIPT_DIR/$1.dockerfile" \ --build-arg DOCKER_NAMESPACE=$DOCKER_USERNAME \ diff --git a/dockerfiles/vcpkg/centos-7.dockerfile b/dockerfiles/vcpkg/centos-7.dockerfile deleted file mode 100644 index 7369d75cdcc..00000000000 --- a/dockerfiles/vcpkg/centos-7.dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF - -RUN yum makecache && yum install -y \ - epel-release \ - java-11-openjdk-devel \ - python3-devel \ - wget && \ - yum update -y && yum install -y \ - ccache \ - R-core-devel && \ - yum -y clean all && rm -rf /var/cache - -ENV Rcpp_package=Rcpp_0.12.19.tar.gz -ENV RInside_package=RInside_0.2.12.tar.gz - -RUN wget https://cran.r-project.org/src/contrib/Archive/Rcpp/${Rcpp_package} -RUN wget https://cran.r-project.org/src/contrib/Archive/RInside/${RInside_package} -RUN R CMD INSTALL ${Rcpp_package} ${RInside_package} -RUN rm -f ${Rcpp_package} ${RInside_package} - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/centos-8.dockerfile b/dockerfiles/vcpkg/centos-8.dockerfile deleted file mode 100644 index 35d54ef9567..00000000000 --- a/dockerfiles/vcpkg/centos-8.dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-centos-8:$VCPKG_REF - -RUN yum remove -y python3.11 java-1.* && yum install -y \ - java-11-openjdk-devel \ - python3-devel \ - epel-release && \ - yum update -y && yum install -y \ - ccache \ - R-core-devel \ - R-Rcpp-devel \ - R-RInside-devel - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/ubuntu-20.04.dockerfile b/dockerfiles/vcpkg/ubuntu-20.04.dockerfile deleted file mode 100644 index c5d0585c472..00000000000 --- a/dockerfiles/vcpkg/ubuntu-20.04.dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-ubuntu-20.04:$VCPKG_REF - -ENV RInside_package=RInside_0.2.14.tar.gz - -RUN apt-get update && apt-get install --no-install-recommends -y \ - ccache \ - default-jdk \ - python3-dev \ - wget \ - r-base \ - r-cran-rcpp -RUN wget https://cran.r-project.org/src/contrib/Archive/RInside/${RInside_package} -RUN R CMD INSTALL ${RInside_package} -RUN rm -f ${RInside_package} - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/ubuntu-22.04.dockerfile deleted file mode 100644 index 9ff881b9bb3..00000000000 --- a/dockerfiles/vcpkg/ubuntu-22.04.dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-ubuntu-22.04:$VCPKG_REF - -RUN apt-get update && apt-get install --no-install-recommends -y \ - ccache \ - default-jdk \ - ninja-build \ - python3-dev \ - rsync \ - fop \ - libsaxonb-java \ - r-base \ - r-cran-rcpp \ - r-cran-rinside \ - r-cran-inline - -RUN git config --global --add safe.directory '*' - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/dockerfiles/vcpkg/ubuntu-23.10.dockerfile b/dockerfiles/vcpkg/ubuntu-23.10.dockerfile deleted file mode 100644 index a9bbac10c3d..00000000000 --- a/dockerfiles/vcpkg/ubuntu-23.10.dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -ARG VCPKG_REF=latest -FROM hpccsystems/platform-build-base-ubuntu-23.10:$VCPKG_REF - -RUN apt-get update && apt-get install --no-install-recommends -y \ - ccache \ - default-jdk \ - ninja-build \ - python3-dev \ - rsync \ - fop \ - libsaxonb-java \ - r-base \ - r-cran-rcpp \ - r-cran-rinside \ - r-cran-inline - -RUN git config --global --add safe.directory '*' - -WORKDIR /hpcc-dev - -ENTRYPOINT ["/bin/bash", "--login", "-c"] diff --git a/vcpkg b/vcpkg index fcde93bcb3f..a65a37760ac 160000 --- a/vcpkg +++ b/vcpkg @@ -1 +1 @@ -Subproject commit fcde93bcb3fec4ae8b650cec2e4e1b7787308098 +Subproject commit a65a37760ac2a85527b72b2d81aa50d882ec2e70