HPCC-29914 Add embedded wasm support

Signed-off-by: Gordon Smith <GordonJSmith@gmail.com>

.github/workflows/build-assets.yml

@@ -55,7 +55,7 @@ jobs:
           echo "internal_tag=$(echo $community_tag | sed 's/community/internal/')" >> $GITHUB_OUTPUT
           community_base_ref=${{ github.event.base_ref || github.ref }}
           echo "community_branch=$(echo $community_base_ref | cut -d'/' -f3)" >> $GITHUB_OUTPUT
-          echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DSIGN_MODULES=ON" >> $GITHUB_OUTPUT
+          echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DSIGN_MODULES=ON" >> $GITHUB_OUTPUT
           echo 'gpg_import=gpg --batch --import /hpcc-dev/build/private.key' >> $GITHUB_OUTPUT
 
       - name: Print vars
@@ -155,12 +155,16 @@ jobs:
           tags: |
             build-${{ matrix.os }}:latest
 
+      - name: Prepare build folder
+        run: |
+          mkdir -p ${{ needs.preamble.outputs.folder_build }}
+          docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cp /hpcc-dev/vcpkg_installed /hpcc-dev/build" 
+
       - name: CMake Packages
         if: ${{ !matrix.container && !matrix.ln && !matrix.documentation }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
-          plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM")
+          plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM")
           for plugin in "${plugins[@]}"; do
             sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
             sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
@@ -175,7 +179,6 @@ jobs:
       - name: CMake Containerized Packages
         if: ${{ matrix.container }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
@@ -209,7 +212,6 @@ jobs:
       - name: CMake LN Packages
         if: ${{ matrix.ln }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           echo "${{ secrets.SIGNING_SECRET }}" > ${{ needs.preamble.outputs.folder_build }}/private.key
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles

.github/workflows/build-vcpkg.yml

@@ -58,7 +58,7 @@ jobs:
           echo "internal_tag=$(echo $community_tag | sed 's/community/internal/')" >> $GITHUB_OUTPUT
           community_base_ref=${{ github.event.base_ref || github.ref }}
           echo "community_branch=$(echo $community_base_ref | cut -d'/' -f3)" >> $GITHUB_OUTPUT
-          echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" >> $GITHUB_OUTPUT
+          echo "cmake_docker_config=-DCMAKE_BUILD_TYPE=RelWithDebInfo -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF" >> $GITHUB_OUTPUT
 
       - id: skip_check
         uses: hpcc-systems/github-actions/changed-modules@main
@@ -147,13 +147,17 @@ jobs:
           tags: |
             build-${{ matrix.os }}:latest
 
+      - name: Prepare build folder
+        run: |
+          mkdir -p ${{ needs.preamble.outputs.folder_build }}
+          docker run --rm --mount ${{ needs.preamble.outputs.mount_platform }} --mount ${{ needs.preamble.outputs.mount_build }} $docker_label "cp /hpcc-dev/vcpkg_installed /hpcc-dev/build" 
+
       - name: CMake Packages
         if: ${{ !matrix.container && !matrix.ln && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           declare -a plugins
           if [ ${{ needs.preamble.outputs.include_plugins }} == "ON" ]; then
-              plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "PLATFORM")
+              plugins=("CASSANDRAEMBED" "COUCHBASEEMBED" "ECLBLAS" "H3" "JAVAEMBED" "KAFKA" "MEMCACHED" "MONGODBEMBED" "MYSQLEMBED" "NLP" "REDIS" "REMBED" "SQLITE3EMBED" "SQS" "WASMEMBED" "PLATFORM")
           else
               plugins=("PLATFORM")
           fi
@@ -171,7 +175,6 @@ jobs:
       - name: CMake Containerized Packages
         if: ${{ matrix.container && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
           docker_label=build-${{ matrix.os }}:latest
@@ -182,7 +185,6 @@ jobs:
       - name: CMake LN Packages
         if: ${{ matrix.ln && contains(matrix.event_name, github.event_name) && needs.preamble.outputs.platform }}
         run: |
-          mkdir -p ${{ needs.preamble.outputs.folder_build }}
           sudo rm -f ${{ needs.preamble.outputs.folder_build }}/CMakeCache.txt
           sudo rm -rf ${{ needs.preamble.outputs.folder_build }}/CMakeFiles
           docker_label=build-${{ matrix.os }}:latest

CMakeLists.txt

@@ -169,6 +169,7 @@ if ( PLUGIN )
     HPCC_ADD_SUBDIRECTORY (dali/base)
     HPCC_ADD_SUBDIRECTORY (plugins/Rembed "REMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/v8embed "V8EMBED")
+    HPCC_ADD_SUBDIRECTORY (plugins/wasmembed "WASMEMBED")
     HPCC_ADD_SUBDIRECTORY (plugins/memcached "MEMCACHED")
     HPCC_ADD_SUBDIRECTORY (plugins/redis "REDIS")
     HPCC_ADD_SUBDIRECTORY (plugins/javaembed "JAVAEMBED")

cmake_modules/plugins.cmake

@@ -41,6 +41,7 @@ set(PLUGINS_LIST
     SQLITE3EMBED
     SQS
     V8EMBED
+    WASMEMBED
     EXAMPLEPLUGIN
 )
 

dockerfiles/vcpkg/build.sh

@@ -25,22 +25,28 @@ echo "DOCKER_PASSWORD: $DOCKER_PASSWORD"
 # docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
 
 function doBuild() {
-    docker build --progress plain --pull --rm -f "$SCRIPT_DIR/$1.dockerfile" \
+    docker build --progress plain -f "$SCRIPT_DIR/$1/Dockerfile" \
         -t build-$1:$GITHUB_REF \
         -t build-$1:latest \
         --build-arg DOCKER_NAMESPACE=$DOCKER_USERNAME \
         --build-arg VCPKG_REF=$VCPKG_REF \
-        "$SCRIPT_DIR/." 
+        "$SCRIPT_DIR/$1/." 
+
+    mkdir -p build-$1
+
+    docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-$1:$GITHUB_REF \
+        "mkdir -p /hpcc-dev/HPCC-Platform/build-$1 && cp -r /hpcc-dev/vcpkg_installed /hpcc-dev/HPCC-Platform/build-$1/vcpkg_installed"
 
     docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-$1:$GITHUB_REF \
         "cmake -S /hpcc-dev/HPCC-Platform -B /hpcc-dev/HPCC-Platform/build-$1 ${CMAKE_OPTIONS}"
+
     docker run --rm --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-$1:$GITHUB_REF \
         "cmake --build /hpcc-dev/HPCC-Platform/build-$1 --parallel $(nproc)"
 
 # docker run -it --mount source="$(pwd)",target=/hpcc-dev/HPCC-Platform,type=bind,consistency=cached build-ubuntu-22.04:latest bash
 }
 
-CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DVCPKG_FILES_DIR=/hpcc-dev -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON"
+CMAKE_OPTIONS="-G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCPACK_THREADS=0 -DUSE_OPTIONAL=OFF -DINCLUDE_PLUGINS=ON -DSUPPRESS_V8EMBED=ON"
 
 doBuild centos-7
 doBuild centos-8

dockerfiles/vcpkg/centos-7.dockerfile → dockerfiles/vcpkg/centos-7/Dockerfile

@@ -1,11 +1,12 @@
 ARG VCPKG_REF=latest
-FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF
+FROM hpccsystems/platform-build-base-centos-7:$VCPKG_REF 
 
 RUN yum install -y \
     java-11-openjdk-devel \
     python3-devel \
     wget \
     epel-release
+
 RUN yum update -y && yum install -y R-core-devel
 
 ENV Rcpp_package=Rcpp_0.12.19.tar.gz

dockerfiles/vcpkg/centos-8.dockerfile → dockerfiles/vcpkg/centos-8/Dockerfile

@@ -5,6 +5,7 @@ RUN yum remove -y java-1.* && yum install -y \
     java-11-openjdk-devel \
     python3-devel \
     epel-release
+
 RUN yum install -y \
     R-core-devel \
     R-Rcpp-devel \

dockerfiles/vcpkg/ubuntu-20.04.dockerfile → dockerfiles/vcpkg/ubuntu-20.04/Dockerfile

@@ -9,8 +9,11 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     wget \
     r-base \
     r-cran-rcpp
+
 RUN wget https://cran.r-project.org/src/contrib/Archive/RInside/${RInside_package}
+
 RUN R CMD INSTALL ${RInside_package}
+
 RUN rm -f ${RInside_package}
 
 WORKDIR /hpcc-dev

plugins/CMakeLists.txt

@@ -30,6 +30,7 @@ add_subdirectory (proxies)
 add_subdirectory (sqlite3)
 add_subdirectory (mysql)
 add_subdirectory (v8embed)
+add_subdirectory (wasmembed)
 HPCC_ADD_SUBDIRECTORY (py3embed "USE_PYTHON3")
 HPCC_ADD_SUBDIRECTORY (pyembed "USE_PYTHON2")
 add_subdirectory (javaembed)

plugins/wasmembed/CMakeLists.txt

@@ -0,0 +1,48 @@
+project(wasmembed)
+
+if(WASMEMBED)
+    ADD_PLUGIN(wasmembed)
+    if(MAKE_WASMEMBED)
+
+        add_subdirectory(secure-enclave)
+
+        include_directories(
+            ./../../common/thorhelper
+            ./../../dali/base
+            ./../../rtl/eclrtl
+            ./../../rtl/include
+            ./../../rtl/nbcd
+            ./../../system/include
+            ./../../system/jlib
+            ./../../system/mp
+        )
+
+        add_definitions(-D_USRDLL -DWASMEMBED_EXPORTS)
+
+        add_library(wasmembed SHARED 
+            wasmembed.cpp
+        )
+
+        target_link_libraries(wasmembed
+            roxiemem
+            eclrtl
+            jlib
+            secure-enclave
+        )
+
+        install(
+            TARGETS wasmembed 
+            DESTINATION plugins
+        )
+
+    else()
+        message(WARNING "Cannot build wasmembed plugin")
+    endif()
+endif()
+
+if(PLATFORM OR CLIENTTOOLS_ONLY)
+    install(
+        FILES ${CMAKE_CURRENT_SOURCE_DIR}/wasm.ecllib
+        DESTINATION plugins
+        COMPONENT Runtime)
+endif()

plugins/wasmembed/secure-enclave/CMakeLists.txt

@@ -0,0 +1,41 @@
+project(secure-enclave)
+
+set(CMAKE_CXX_STANDARD 20)
+
+find_path(WASMTIME_CPP_API_INCLUDE_DIRS "wasmtime-cpp-api/wasmtime.hh"
+    PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+)
+if (WIN32)
+    find_library(WASMTIME_LIB NAMES wasmtime.dll
+        PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+    )
+else()
+    find_library(WASMTIME_LIB NAMES wasmtime
+        PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}
+    )
+endif()
+
+include_directories(
+    ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-c-api
+    ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-cpp-api
+    ./../../../system/include
+    ./../../../rtl/eclrtl
+    ./../../../system/jlib
+)
+
+add_definitions(-D_USRDLL -DSECUREENCLAVE_EXPORTS)
+
+add_library(secure-enclave SHARED
+    secure-enclave.cpp
+)
+
+target_link_libraries(secure-enclave PRIVATE
+    ${WASMTIME_LIB}
+)
+
+install(
+    TARGETS secure-enclave 
+    DESTINATION plugins
+    CALC_DEPS
+)
+