From ed298a8e604fdf097c315d646f759cccbe609ba8 Mon Sep 17 00:00:00 2001 From: Gordon Smith Date: Thu, 13 Jul 2023 12:13:43 +0100 Subject: [PATCH] HPCC-29914 Add embedded wasm support Move definitions into their own cpp file(s) Added a thread safe map Removed std::runtime usage Refactored up some string/unicode calls Revisited ABI code Signed-off-by: Gordon Smith --- plugins/wasmembed/CMakeLists.txt | 22 +- plugins/wasmembed/abi.cpp | 268 +++++++ plugins/wasmembed/abi.hpp | 12 + .../{secure-enclave => }/hpcc-platform.wit | 0 plugins/wasmembed/secure-enclave.cpp | 671 ++++++++++++++++++ plugins/wasmembed/secure-enclave.hpp | 13 + .../wasmembed/secure-enclave/CMakeLists.txt | 41 -- plugins/wasmembed/secure-enclave/abi.hpp | 157 ---- .../secure-enclave/secure-enclave.cpp | 502 ------------- .../secure-enclave/secure-enclave.hpp | 28 - plugins/wasmembed/util.cpp | 59 ++ plugins/wasmembed/util.hpp | 7 + plugins/wasmembed/wasmembed.cpp | 95 +-- system/jlib/jarray.hpp | 2 +- system/jlib/jbuff.hpp | 8 +- system/jlib/jhash.hpp | 6 +- system/jlib/jiface.hpp | 4 +- system/jlib/jsuperhash.hpp | 26 +- testing/regress/ecl/key/wasmembed.xml | 51 ++ testing/regress/ecl/wasmembed.ecl | 81 ++- testing/regress/ecl/wasmembed.wasm | Bin 53462 -> 53388 bytes vcpkg | 2 +- vcpkg_overlays/wasmtime-c-api/portfile.cmake | 6 +- vcpkg_overlays/wasmtime-c-api/vcpkg.json | 4 +- vcpkg_overlays/wasmtime-cpp-api/vcpkg.json | 2 +- 25 files changed, 1210 insertions(+), 857 deletions(-) create mode 100644 plugins/wasmembed/abi.cpp create mode 100644 plugins/wasmembed/abi.hpp rename plugins/wasmembed/{secure-enclave => }/hpcc-platform.wit (100%) create mode 100644 plugins/wasmembed/secure-enclave.cpp create mode 100644 plugins/wasmembed/secure-enclave.hpp delete mode 100644 plugins/wasmembed/secure-enclave/CMakeLists.txt delete mode 100644 plugins/wasmembed/secure-enclave/abi.hpp delete mode 100644 plugins/wasmembed/secure-enclave/secure-enclave.cpp delete mode 100644 plugins/wasmembed/secure-enclave/secure-enclave.hpp create mode 100644 plugins/wasmembed/util.cpp create mode 100644 plugins/wasmembed/util.hpp diff --git a/plugins/wasmembed/CMakeLists.txt b/plugins/wasmembed/CMakeLists.txt index 476fdbf18de..ed5e2216bf0 100644 --- a/plugins/wasmembed/CMakeLists.txt +++ b/plugins/wasmembed/CMakeLists.txt @@ -1,12 +1,27 @@ project(wasmembed) +set(CMAKE_CXX_STANDARD 20) + if(WASMEMBED) ADD_PLUGIN(wasmembed) if(MAKE_WASMEMBED) - add_subdirectory(secure-enclave) + find_path(WASMTIME_CPP_API_INCLUDE_DIRS "wasmtime-cpp-api/wasmtime.hh" + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + if (WIN32) + find_library(WASMTIME_LIB NAMES wasmtime.dll + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + else() + find_library(WASMTIME_LIB NAMES wasmtime + PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} + ) + endif() include_directories( + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-c-api + ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-cpp-api ./../../common/thorhelper ./../../dali/base ./../../rtl/eclrtl @@ -21,13 +36,16 @@ if(WASMEMBED) add_library(wasmembed SHARED wasmembed.cpp + secure-enclave.cpp + abi.cpp + util.cpp ) target_link_libraries(wasmembed + ${WASMTIME_LIB} roxiemem eclrtl jlib - secure-enclave ) install( diff --git a/plugins/wasmembed/abi.cpp b/plugins/wasmembed/abi.cpp new file mode 100644 index 00000000000..d700d3b1496 --- /dev/null +++ b/plugins/wasmembed/abi.cpp @@ -0,0 +1,268 @@ +/* + See: https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md + https://github.com/WebAssembly/component-model/blob/main/design/mvp/canonical-abi/definitions.py +*/ + +#include "abi.hpp" +#include "jexcept.hpp" + +#include +#include +#include +#include +#include + +auto UTF16_TAG = 1 << 31; + +// +/* canonical despecialize (python) ------------------------------------------------------------- + +def despecialize(t): + match t: + case Tuple(ts) : return Record([ Field(str(i), t) for i,t in enumerate(ts) ]) + case Union(ts) : return Variant([ Case(str(i), t) for i,t in enumerate(ts) ]) + case Enum(labels) : return Variant([ Case(l, None) for l in labels ]) + case Option(t) : return Variant([ Case("none", None), Case("some", t) ]) + case Result(ok, error) : return Variant([ Case("ok", ok), Case("error", error) ]) + case _ : return t + +*/ + +// template +// wasmtime::ValType despecialize(const T &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F32: +// case wasmtime::ValKind::F64: +// case wasmtime::ValKind::V128: +// return t.kind(); +// default: +// return wasmtime::ValType::i32(); +// } +// } + +/* canonical alignment (python) ------------------------------------------------------------- + +def alignment(t): + match despecialize(t): + case Bool() : return 1 + case S8() | U8() : return 1 + case S16() | U16() : return 2 + case S32() | U32() : return 4 + case S64() | U64() : return 8 + case Float32() : return 4 + case Float64() : return 8 + case Char() : return 4 + case String() | List(_) : return 4 + case Record(fields) : return alignment_record(fields) + case Variant(cases) : return alignment_variant(cases) + case Flags(labels) : return alignment_flags(labels) + case Own(_) | Borrow(_) : return 4 + +*/ + +// int alignment(const wasmtime::ValType &t) +// { +// switch (t.kind()) +// { +// case wasmtime::ValKind::I32: +// case wasmtime::ValKind::F32: +// return 4; +// case wasmtime::ValKind::I64: +// case wasmtime::ValKind::F64: +// return 8; +// case wasmtime::ValKind::V128: +// return 16; +// default: +// return 1; +// } +// } + +/* canonical align_to (python) ------------------------------------------------------------- + +def align_to(ptr, alignment): + return math.ceil(ptr / alignment) * alignment + +*/ + +uint32_t align_to(uint32_t ptr, uint32_t alignment) +{ + return std::ceil(ptr / alignment) * alignment; +} + +// loading --- + +/* canonical load_int (python) ------------------------------------------------------------- + +def load_int(cx, ptr, nbytes, signed = False): + return int.from_bytes(cx.opts.memory[ptr : ptr+nbytes], 'little', signed=signed) + +*/ + +#include +#include + +// int load_int(const wasmtime::Span &cx, int32_t *ptr, int nbytes, int signed) +// { +// int result = 0; +// for (int i = 0; i < nbytes; i++) { +// result += ptr[i] << (8*i); +// } +// if (signed) { +// result += (ptr[nbytes-1] & 0x80) ? -1 << (8*nbytes) : 0; +// } +// return result; +// } + +template +T load_int(const wasmtime::Span &data, int32_t ptr) +{ + T retVal = 0; + auto nbytes = sizeof(retVal); + for (int i = 0; i < nbytes; ++i) + { + uint8_t b = data[ptr + i]; + retVal += b << (i * 8); + } + if (std::is_signed::value) + { + retVal += (data[ptr + nbytes - 1] & 0x80) ? -1 << (8 * nbytes) : 0; + } + return retVal; +} + +/* canonical load_string_from_range (python) ------------------------------------------------------------- + +def load_string_from_range(cx, ptr, tagged_code_units): + match cx.opts.string_encoding: + case 'utf8': + alignment = 1 + byte_length = tagged_code_units + encoding = 'utf-8' + case 'utf16': + alignment = 2 + byte_length = 2 * tagged_code_units + encoding = 'utf-16-le' + case 'latin1+utf16': + alignment = 2 + if bool(tagged_code_units & UTF16_TAG): + byte_length = 2 * (tagged_code_units ^ UTF16_TAG) + encoding = 'utf-16-le' + else: + byte_length = tagged_code_units + encoding = 'latin-1' + + trap_if(ptr != align_to(ptr, alignment)) + trap_if(ptr + byte_length > len(cx.opts.memory)) + try: + s = cx.opts.memory[ptr : ptr+byte_length].decode(encoding) + except UnicodeError: + trap() + + return (s, cx.opts.string_encoding, tagged_code_units) + +*/ + +// More: Not currently available from the wasmtime::context object, see https://github.com/bytecodealliance/wasmtime/issues/6719 +std::string global_encoding = "utf8"; + +std::pair load_string_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t tagged_code_units) +{ + std::string encoding = "utf-8"; + uint32_t byte_length = tagged_code_units; + uint32_t alignment = 1; + if (global_encoding.compare("utf8") == 0) + { + alignment = 1; + byte_length = tagged_code_units; + encoding = "utf-8"; + } + else if (global_encoding.compare("utf16") == 0) + { + alignment = 2; + byte_length = 2 * tagged_code_units; + encoding = "utf-16-le"; + } + else if (global_encoding.compare("latin1+utf16") == 0) + { + alignment = 2; + if (tagged_code_units & UTF16_TAG) + { + byte_length = 2 * (tagged_code_units ^ UTF16_TAG); + encoding = "utf-16-le"; + } + else + { + byte_length = tagged_code_units; + encoding = "latin-1"; + } + } + + if (ptr != align_to(ptr, alignment)) + { + throw makeStringException(0, "Invalid alignment"); + } + + if (ptr + byte_length > data.size()) + { + throw makeStringException(1, "Out of bounds"); + } + + return std::make_pair(ptr, byte_length); +} + +/* canonical load_string (python) ------------------------------------------------------------- + +def load_string(cx, ptr): + begin = load_int(cx, ptr, 4) + tagged_code_units = load_int(cx, ptr + 4, 4) + return load_string_from_range(cx, begin, tagged_code_units) + +*/ +std::pair load_string(const wasmtime::Span &data, uint32_t ptr) +{ + uint32_t begin = load_int(data, ptr); + uint32_t tagged_code_units = load_int(data, ptr + 4); + return load_string_from_range(data, begin, tagged_code_units); +} + +/* canonical load_list_from_range (python) ------------------------------------------------------------- + +def load_list_from_range(cx, ptr, length, elem_type): + trap_if(ptr != align_to(ptr, alignment(elem_type))) + trap_if(ptr + length * size(elem_type) > len(cx.opts.memory)) + a = [] + for i in range(length): + a.append(load(cx, ptr + i * size(elem_type), elem_type)) + return a + +*/ + +template +std::vector load_list_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t length) +{ + if (ptr != align_to(ptr, alignment(T{}))) + throw makeStringException(2, "Pointer is not aligned"); + if (ptr + length * sizeof(T) > data.size()) + throw makeStringException(1, "Out of bounds access"); + std::vector a; + for (uint32_t i = 0; i < length; i++) + { + a.push_back(load(data, ptr + i * sizeof(T))); + } + return a; +} + +/* canonical load_list (python) ------------------------------------------------------------- + +def load_list(cx, ptr, elem_type): + begin = load_int(cx, ptr, 4) + length = load_int(cx, ptr + 4, 4) + return load_list_from_range(cx, begin, length, elem_type) + +*/ + +// Storing --- diff --git a/plugins/wasmembed/abi.hpp b/plugins/wasmembed/abi.hpp new file mode 100644 index 00000000000..6b27fe85350 --- /dev/null +++ b/plugins/wasmembed/abi.hpp @@ -0,0 +1,12 @@ +#include + +// ABI +int align_to(int ptr, int alignment); + +// ABI Loading --- +template +T load_int(const wasmtime::Span &data, int32_t ptr); + +std::pair load_string(const wasmtime::Span &data, uint32_t ptr); + +// ABI Storing --- diff --git a/plugins/wasmembed/secure-enclave/hpcc-platform.wit b/plugins/wasmembed/hpcc-platform.wit similarity index 100% rename from plugins/wasmembed/secure-enclave/hpcc-platform.wit rename to plugins/wasmembed/hpcc-platform.wit diff --git a/plugins/wasmembed/secure-enclave.cpp b/plugins/wasmembed/secure-enclave.cpp new file mode 100644 index 00000000000..0aab098ab0c --- /dev/null +++ b/plugins/wasmembed/secure-enclave.cpp @@ -0,0 +1,671 @@ +#include "secure-enclave.hpp" +#include "eclrtl.hpp" +#include "eclrtl_imp.hpp" +#include "rtlconst.hpp" +#include "jiface.hpp" +#include "jlog.hpp" +#include "jexcept.hpp" + +// From deftype.hpp in common +#define UNKNOWN_LENGTH 0xFFFFFFF1 + +#include "abi.hpp" +#include "util.hpp" + +#include +#include +#include +#include + +// #define ENABLE_TRACE +#ifdef ENABLE_TRACE +#define TRACE(format, ...) DBGLOG(format, ##__VA_ARGS__) +#else +#define TRACE(format, ...) \ + do \ + { \ + } while (0) +#endif + +template +class ThreadSafeMap +{ +protected: + std::unordered_map map; + mutable std::shared_mutex mutex; + +public: + ThreadSafeMap() {} + ~ThreadSafeMap() {} + + void clear() + { + TRACE("WASM SE ThreadSafeMap clear"); + std::unique_lock lock(mutex); + map.clear(); + TRACE("WASM SE ThreadSafeMap clear2"); + } + + void insertIfMissing(const K &key, std::function &valueCallback) + { + TRACE("WASM SE ThreadSafeMap insertIfMissing"); + std::unique_lock lock(mutex); + if (map.find(key) == map.end()) + map.insert(std::make_pair(key, valueCallback())); + TRACE("WASM SE ThreadSafeMap insertIfMissing2"); + } + + void erase(const K &key) + { + TRACE("WASM SE ThreadSafeMap erase"); + std::unique_lock lock(mutex); + map.erase(key); + TRACE("WASM SE ThreadSafeMap erase2"); + } + + bool find(const K &key, std::optional &value) const + { + TRACE("WASM SE ThreadSafeMap find"); + std::shared_lock lock(mutex); + auto it = map.find(key); + if (it != map.end()) + { + value = it->second; + TRACE("WASM SE ThreadSafeMap find2"); + return true; + } + TRACE("WASM SE ThreadSafeMap find3"); + return false; + } + + bool has(const K &key) const + { + TRACE("WASM SE ThreadSafeMap has"); + std::shared_lock lock(mutex); + TRACE("WASM SE ThreadSafeMap has2"); + return map.find(key) != map.end(); + } + + void for_each(std::function func) const + { + TRACE("WASM SE ThreadSafeMap for_each"); + std::shared_lock lock(mutex); + for (auto it = map.begin(); it != map.end(); ++it) + { + func(it->first, it->second); + } + TRACE("WASM SE ThreadSafeMap for_each2"); + } +}; + +class WasmEngine +{ +private: + wasmtime::Engine engine; + wasmtime::Store store; + + ThreadSafeMap wasmInstances; + // wasmMems and wasmFuncs are only written to during createInstance, so no need for a mutex + std::unordered_map wasmMems; + std::unordered_map wasmFuncs; + +public: + WasmEngine() : store(engine) + { + TRACE("WASM SE WasmEngine"); + } + + ~WasmEngine() + { + TRACE("WASM SE ~WasmEngine"); + } + + bool hasInstance(const std::string &wasmName) const + { + TRACE("WASM SE hasInstance"); + return wasmInstances.has(wasmName); + } + + wasmtime::Instance getInstance(const std::string &wasmName) const + { + TRACE("WASM SE getInstance"); + std::optional instance; + if (!wasmInstances.find(wasmName, instance)) + throw makeStringExceptionV(-1, "Wasm instance not found: %s", wasmName.c_str()); + return instance.value(); + } + + wasmtime::Instance createInstance(const std::string &wasmName, const std::variant> &wasm) + { + TRACE("WASM SE resolveModule %s", wasmName.c_str()); + auto module = std::holds_alternative(wasm) ? wasmtime::Module::compile(engine, std::get(wasm)).unwrap() : wasmtime::Module::compile(engine, std::get>(wasm)).unwrap(); + TRACE("WASM SE resolveModule2 %s", wasmName.c_str()); + + wasmtime::WasiConfig wasi; + wasi.inherit_argv(); + wasi.inherit_env(); + wasi.inherit_stdin(); + wasi.inherit_stdout(); + wasi.inherit_stderr(); + store.context().set_wasi(std::move(wasi)).unwrap(); + TRACE("WASM SE resolveModule3 %s", wasmName.c_str()); + + wasmtime::Linker linker(engine); + linker.define_wasi().unwrap(); + TRACE("WASM SE resolveModule4 %s", wasmName.c_str()); + + auto callback = [this, wasmName](wasmtime::Caller caller, uint32_t msg, uint32_t msg_len) + { + TRACE("WASM SE callback: %i %i", msg_len, msg); + auto data = this->getData(wasmName); + auto msg_ptr = (const char *)&data[msg]; + DBGLOG("from wasm: %.*s", msg_len, msg_ptr); + }; + auto host_func = linker.func_wrap("$root", "dbglog", callback).unwrap(); + + auto newInstance = linker.instantiate(store, module).unwrap(); + linker.define_instance(store, "linking2", newInstance).unwrap(); + + for (auto exportItem : module.exports()) + { + auto externType = wasmtime::ExternType::from_export(exportItem); + std::string name(exportItem.name()); + if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported function: %s", name.c_str()); + auto func = std::get(*newInstance.get(store, name)); + wasmFuncs.insert(std::make_pair(wasmName + "." + name, func)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported memory: %s", name.c_str()); + auto memory = std::get(*newInstance.get(store, name)); + wasmMems.insert(std::make_pair(wasmName + "." + name, memory)); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported table: %s", name.c_str()); + } + else if (std::holds_alternative(externType)) + { + TRACE("WASM SE Exported global: %s", name.c_str()); + } + else + { + TRACE("WASM SE Unknown export type"); + } + } + + return newInstance; + } + + void registerInstance(const std::string &wasmName, const std::variant> &wasm) + { + TRACE("WASM SE registerInstance %s", wasmName.c_str()); + std::function createInstanceCallback = [this, wasmName, wasm]() + { + return createInstance(wasmName, wasm); + }; + wasmInstances.insertIfMissing(wasmName, createInstanceCallback); + } + + bool hasFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE hasFunc"); + return wasmFuncs.find(qualifiedID) != wasmFuncs.end(); + } + + wasmtime::Func getFunc(const std::string &qualifiedID) const + { + TRACE("WASM SE getFunc"); + auto found = wasmFuncs.find(qualifiedID); + if (found == wasmFuncs.end()) + throw makeStringExceptionV(-1, "Wasm function not found: %s", qualifiedID.c_str()); + TRACE("WASM SE getFunc2"); + return found->second; + } + + wasmtime::ValType::ListRef getFuncParams(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncParams"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->params(); + } + + wasmtime::ValType::ListRef getFuncResults(const std::string &qualifiedID) + { + TRACE("WASM SE getFuncResults"); + auto func = getFunc(qualifiedID); + wasmtime::FuncType funcType = func.type(store.context()); + return funcType->results(); + } + + std::vector call(const std::string &qualifiedID, const std::vector ¶ms) + { + TRACE("WASM SE call"); + auto func = getFunc(qualifiedID); + auto retVal = func.call(store, params).unwrap(); + TRACE("WASM SE call 2"); + return retVal; + } + + std::vector callRealloc(const std::string &wasmName, const std::vector ¶ms) + { + TRACE("WASM SE callRealloc"); + return call(createQualifiedID(wasmName, "cabi_realloc"), params); + } + + wasmtime::Span getData(const std::string &wasmName) + { + TRACE("WASM SE getData"); + auto found = wasmMems.find(createQualifiedID(wasmName, "memory")); + if (found == wasmMems.end()) + throw makeStringExceptionV(-1, "Wasm memory not found: %s", wasmName.c_str()); + return found->second.data(store.context()); + } +}; +static std::unique_ptr wasmEngine; + +class SecureFunction : public CInterfaceOf +{ + std::string wasmName; + std::string funcName; + std::string qualifiedID; + + const IThorActivityContext *activityCtx = nullptr; + std::vector args; + std::vector wasmResults; + + StringArray manifestModules; + +public: + SecureFunction(const StringArray &_manifestModules) + { + TRACE("WASM SE se:constructor"); + manifestModules.appendArray(_manifestModules); + if (!wasmEngine) + { + TRACE("WASM SE se:constructor2"); + wasmEngine = std::make_unique(); + } + TRACE("WASM SE se:constructor3"); + } + + virtual ~SecureFunction() + { + TRACE("WASM SE se:destructor"); + + // Garbage Collection --- + // Function results --- + auto gc_func_name = createQualifiedID(wasmName, "cabi_post_" + funcName); + TRACE("WASM SE se:destructor %s", gc_func_name.c_str()); + if (wasmEngine->hasFunc(gc_func_name)) + { + for (auto &result : wasmResults) + { + wasmEngine->call(gc_func_name, {result}); + } + } + } + + const char *resolveManifestPath(const char *leafName) + { + if (leafName && *leafName) + { + ForEachItemIn(idx, manifestModules) + { + const char *path = manifestModules.item(idx); + if (endsWith(path, leafName)) + return path; + } + } + return nullptr; + } + + // IEmbedFunctionContext --- + void setActivityContext(const IThorActivityContext *_activityCtx) + { + activityCtx = _activityCtx; + } + + virtual IInterface *bindParamWriter(IInterface *esdl, const char *esdlservice, const char *esdltype, const char *name) + { + TRACE("WASM SE paramWriterCommit"); + return NULL; + } + virtual void paramWriterCommit(IInterface *writer) + { + TRACE("WASM SE paramWriterCommit"); + } + virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer) + { + TRACE("WASM SE writeResult"); + } + virtual void bindBooleanParam(const char *name, bool val) + { + TRACE("WASM SE bindBooleanParam %s %i", name, val); + args.push_back(val); + } + virtual void bindDataParam(const char *name, size32_t len, const void *val) + { + TRACE("WASM SE bindDataParam %s %d", name, len); + } + virtual void bindFloatParam(const char *name, float val) + { + TRACE("WASM SE bindFloatParam %s %f", name, val); + args.push_back(val); + } + virtual void bindRealParam(const char *name, double val) + { + TRACE("WASM SE bindRealParam %s %f", name, val); + args.push_back(val); + } + virtual void bindSignedSizeParam(const char *name, int size, __int64 val) + { + TRACE("WASM SE bindSignedSizeParam %s %i %lld", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindSignedParam(const char *name, __int64 val) + { + TRACE("WASM SE bindSignedParam %s %lld", name, val); + args.push_back(static_cast(val)); + } + virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedSizeParam %s %i %llu", name, size, val); + if (size <= 4) + args.push_back(static_cast(val)); + else + args.push_back(static_cast(val)); + } + virtual void bindUnsignedParam(const char *name, unsigned __int64 val) + { + TRACE("WASM SE bindUnsignedParam %s %llu", name, val); + args.push_back(static_cast(val)); + } + virtual void bindStringParam(const char *name, size32_t codeUnits, const char *val) + { + TRACE("WASM SE bindStringParam %s %d %s", name, codeUnits, val); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlStrToUtf8X(utfCharCount, utfText.refstr(), codeUnits, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + virtual void bindVStringParam(const char *name, const char *val) + { + TRACE("WASM SE bindVStringParam %s %s", name, val); + bindStringParam(name, strlen(val), val); + } + virtual void bindUTF8Param(const char *name, size32_t codePoints, const char *val) + { + TRACE("WASM SE bindUTF8Param %s %d %s", name, codePoints, val); + auto codeUnits = rtlUtf8Size(codePoints, val); + auto memIdxVar = wasmEngine->callRealloc(wasmName, {0, 0, 1, (int32_t)codeUnits}); + auto memIdx = memIdxVar[0].i32(); + auto mem = wasmEngine->getData(wasmName); + memcpy(&mem[memIdx], val, codeUnits); + args.push_back(memIdx); + args.push_back((int32_t)codeUnits); + } + virtual void bindUnicodeParam(const char *name, size32_t codePoints, const UChar *val) + { + TRACE("WASM SE bindUnicodeParam %s %d", name, codePoints); + size32_t utfCharCount; + rtlDataAttr utfText; + rtlUnicodeToUtf8X(utfCharCount, utfText.refstr(), codePoints, val); + bindUTF8Param(name, utfCharCount, utfText.getstr()); + } + + virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, const void *setData) + { + TRACE("WASM SE bindSetParam %s %d %d %d %d %p", name, elemType, elemSize, isAll, totalBytes, setData); + type_vals typecode = (type_vals)elemType; + const byte *inData = (const byte *)setData; + const byte *endData = inData + totalBytes; + int numElems; + if (elemSize == UNKNOWN_LENGTH) + { + numElems = 0; + // Will need 2 passes to work out how many elements there are in the set :( + while (inData < endData) + { + int thisSize; + switch (elemType) + { + case type_varstring: + thisSize = strlen((const char *)inData) + 1; + break; + case type_string: + thisSize = *(size32_t *)inData + sizeof(size32_t); + break; + case type_unicode: + thisSize = (*(size32_t *)inData) * sizeof(UChar) + sizeof(size32_t); + break; + case type_utf8: + thisSize = rtlUtf8Size(*(size32_t *)inData, inData + sizeof(size32_t)) + sizeof(size32_t); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + inData += thisSize; + numElems++; + } + inData = (const byte *)setData; + } + else + numElems = totalBytes / elemSize; + + std::vector memIdxVar; + int32_t memIdx; + + switch (typecode) + { + case type_boolean: + memIdxVar = wasmEngine->callRealloc(wasmName, {0, 0, 1, (int32_t)numElems}); + memIdx = memIdxVar[0].i32(); + break; + default: + rtlFail(0, "wasmembed: Unsupported parameter type"); + break; + } + + auto mem = wasmEngine->getData(wasmName); + size32_t thisSize = elemSize; + for (int idx = 0; idx < numElems; idx++) + { + switch (typecode) + { + case type_boolean: + mem[memIdx + idx] = *(bool *)inData; + break; + default: + rtlFail(0, "v8embed: Unsupported parameter type"); + break; + } + inData += thisSize; + } + args.push_back(memIdx); + args.push_back(numElems); + } + + virtual void bindRowParam(const char *name, IOutputMetaData &metaVal, const byte *val) override + { + TRACE("WASM SE bindRowParam %s %p", name, val); + throw makeStringException(-1, "bindRowParam not implemented"); + } + virtual void bindDatasetParam(const char *name, IOutputMetaData &metaVal, IRowStream *val) + { + TRACE("WASM SE bindDatasetParam %s %p", name, val); + throw makeStringException(-1, "bindDatasetParam not implemented"); + } + virtual bool getBooleanResult() + { + TRACE("WASM SE getBooleanResult"); + return wasmResults[0].i32(); + } + virtual void getDataResult(size32_t &len, void *&result) + { + TRACE("WASM SE getDataResult"); + throw makeStringException(-1, "getDataResult not implemented"); + } + virtual double getRealResult() + { + TRACE("WASM SE getRealResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::F64) + return wasmResults[0].f64(); + return wasmResults[0].f32(); + } + virtual __int64 getSignedResult() + { + TRACE("WASM SE getSignedResult1 %i", (uint8_t)wasmResults[0].kind()); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + { + return wasmResults[0].i64(); + } + return static_cast<__int64>(wasmResults[0].i32()); + } + virtual unsigned __int64 getUnsignedResult() + { + TRACE("WASM SE getUnsignedResult"); + if (wasmResults[0].kind() == wasmtime::ValKind::I64) + return wasmResults[0].i64(); + return static_cast(wasmResults[0].i32()); + } + virtual void getStringResult(size32_t &chars, char *&result) + { + TRACE("WASM SE getStringResult %zu", wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmEngine->getData(wasmName); + uint32_t strPtr; + uint32_t codeUnits; + std::tie(strPtr, codeUnits) = load_string(data, ptr); + rtlStrToStrX(chars, result, codeUnits, reinterpret_cast(&data[strPtr])); + } + virtual void getUTF8Result(size32_t &chars, char *&result) + { + TRACE("WASM SE getUTF8Result"); + auto ptr = wasmResults[0].i32(); + auto data = wasmEngine->getData(wasmName); + uint32_t strPtr; + uint32_t codeUnits; + std::tie(strPtr, codeUnits) = load_string(data, ptr); + chars = rtlUtf8Length(codeUnits, &data[strPtr]); + TRACE("WASM SE getUTF8Result %d %d", codeUnits, chars); + result = (char *)rtlMalloc(codeUnits); + memcpy(result, &data[strPtr], codeUnits); + } + virtual void getUnicodeResult(size32_t &chars, UChar *&result) + { + TRACE("WASM SE getUnicodeResult"); + auto ptr = wasmResults[0].i32(); + auto data = wasmEngine->getData(wasmName); + uint32_t strPtr; + uint32_t codeUnits; + std::tie(strPtr, codeUnits) = load_string(data, ptr); + unsigned numchars = rtlUtf8Length(codeUnits, &data[strPtr]); + rtlUtf8ToUnicodeX(chars, result, numchars, reinterpret_cast(&data[strPtr])); + } + virtual void getSetResult(bool &__isAllResult, size32_t &resultBytes, void *&result, int elemType, size32_t elemSize) + { + TRACE("WASM SE getSetResult %d %d %zu", elemType, elemSize, wasmResults.size()); + auto ptr = wasmResults[0].i32(); + auto data = wasmEngine->getData(wasmName); + + throw makeStringException(-1, "getSetResult not implemented"); + } + virtual IRowStream *getDatasetResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getDatasetResult"); + throw makeStringException(-1, "getDatasetResult not implemented"); + return NULL; + } + virtual byte *getRowResult(IEngineRowAllocator *_resultAllocator) + { + TRACE("WASM SE getRowResult"); + throw makeStringException(-1, "getRowResult not implemented"); + return NULL; + } + virtual size32_t getTransformResult(ARowBuilder &builder) + { + TRACE("WASM SE getTransformResult"); + throw makeStringException(-1, "getTransformResult not implemented"); + return 0; + } + virtual void loadCompiledScript(size32_t chars, const void *_script) override + { + TRACE("WASM SE loadCompiledScript %p", _script); + throw makeStringException(-1, "loadCompiledScript not implemented"); + } + virtual void enter() override + { + TRACE("WASM SE enter"); + } + virtual void reenter(ICodeContext *codeCtx) override + { + TRACE("WASM SE reenter"); + } + virtual void exit() override + { + TRACE("WASM SE exit"); + } + virtual void compileEmbeddedScript(size32_t lenChars, const char *_utf) override + { + TRACE("WASM SE compileEmbeddedScript"); + std::string utf(_utf, lenChars); + funcName = extractContentInDoubleQuotes(utf); + wasmName = "embed_" + funcName; + qualifiedID = createQualifiedID(wasmName, funcName); + wasmEngine->registerInstance(wasmName, utf); + } + virtual void importFunction(size32_t lenChars, const char *qualifiedName) override + { + TRACE("WASM SE importFunction: %s", qualifiedName); + + qualifiedID = std::string(qualifiedName, lenChars); + auto [_wasmName, _funcName] = splitQualifiedID(qualifiedID); + wasmName = _wasmName; + funcName = _funcName; + + if (!wasmEngine->hasInstance(wasmName)) + { + std::string fullPath = resolveManifestPath((wasmName + ".wasm").c_str()); + auto wasmFile = read_wasm_binary_to_buffer(fullPath); + wasmEngine->registerInstance(wasmName, wasmFile); + } + } + virtual void callFunction() + { + TRACE("WASM SE callFunction %s", qualifiedID.c_str()); + wasmResults = wasmEngine->call(qualifiedID, args); + } +}; + +SECUREENCLAVE_API IEmbedFunctionContext *createISecureEnclave(const StringArray &manifestModules) +{ + // TRACE("WASM SE createISecureEnclave"); + return new SecureFunction(manifestModules); +} + +SECUREENCLAVE_API void syntaxCheck(size32_t &lenResult, char *&result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions) +{ + std::string errMsg = ""; + try + { + wasmtime::Engine engine; + wasmtime::Store store(engine); + auto module = wasmtime::Module::compile(engine, body); + } + catch (const wasmtime::Error &e) + { + errMsg = e.message(); + } + + lenResult = errMsg.length(); + result = reinterpret_cast(rtlMalloc(lenResult)); + errMsg.copy(result, lenResult); +} diff --git a/plugins/wasmembed/secure-enclave.hpp b/plugins/wasmembed/secure-enclave.hpp new file mode 100644 index 00000000000..6deb2739e6a --- /dev/null +++ b/plugins/wasmembed/secure-enclave.hpp @@ -0,0 +1,13 @@ +#include "platform.h" +#include "jutil.hpp" +#include "eclrtl.hpp" + +#ifdef SECUREENCLAVE_EXPORTS +#define SECUREENCLAVE_API DECL_EXPORT +#else +#define SECUREENCLAVE_API DECL_IMPORT +#endif + +SECUREENCLAVE_API IEmbedFunctionContext *createISecureEnclave(const StringArray &manifestModules); + +SECUREENCLAVE_API void syntaxCheck(size32_t &__lenResult, char *&__result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions); diff --git a/plugins/wasmembed/secure-enclave/CMakeLists.txt b/plugins/wasmembed/secure-enclave/CMakeLists.txt deleted file mode 100644 index b4247b79d9b..00000000000 --- a/plugins/wasmembed/secure-enclave/CMakeLists.txt +++ /dev/null @@ -1,41 +0,0 @@ -project(secure-enclave) - -set(CMAKE_CXX_STANDARD 20) - -find_path(WASMTIME_CPP_API_INCLUDE_DIRS "wasmtime-cpp-api/wasmtime.hh" - PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} -) -if (WIN32) - find_library(WASMTIME_LIB NAMES wasmtime.dll - PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} - ) -else() - find_library(WASMTIME_LIB NAMES wasmtime - PATHS ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET} - ) -endif() - -include_directories( - ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-c-api - ${WASMTIME_CPP_API_INCLUDE_DIRS}/wasmtime-cpp-api - ./../../../system/include - ./../../../rtl/eclrtl - ./../../../system/jlib -) - -add_definitions(-D_USRDLL -DSECUREENCLAVE_EXPORTS) - -add_library(secure-enclave SHARED - secure-enclave.cpp -) - -target_link_libraries(secure-enclave PRIVATE - ${WASMTIME_LIB} -) - -install( - TARGETS secure-enclave - DESTINATION plugins - CALC_DEPS -) - diff --git a/plugins/wasmembed/secure-enclave/abi.hpp b/plugins/wasmembed/secure-enclave/abi.hpp deleted file mode 100644 index c547a9fd29a..00000000000 --- a/plugins/wasmembed/secure-enclave/abi.hpp +++ /dev/null @@ -1,157 +0,0 @@ -/* - See: https://github.com/WebAssembly/component-model/blob/main/design/mvp/CanonicalABI.md - https://github.com/WebAssembly/component-model/blob/main/design/mvp/canonical-abi/definitions.py -*/ - -#include -#include -#include -#include -#include - -auto UTF16_TAG = 1 << 31; - -int align_to(int ptr, int alignment) -{ - return std::ceil(ptr / alignment) * alignment; -} - -// loading --- - -int load_int(const wasmtime::Span &data, int32_t ptr, int32_t nbytes, bool is_signed = false) -{ - int result = 0; - for (int i = 0; i < nbytes; i++) - { - int b = data[ptr + i]; - if (i == 3 && is_signed && b >= 0x80) - { - b -= 0x100; - } - result += b << (i * 8); - } - return result; -} - -std::string global_encoding = "utf8"; -std::string load_string_from_range(const wasmtime::Span &data, uint32_t ptr, uint32_t tagged_code_units) -{ - std::string encoding = "utf-8"; - uint32_t byte_length = tagged_code_units; - uint32_t alignment = 1; - if (global_encoding.compare("utf8") == 0) - { - alignment = 1; - byte_length = tagged_code_units; - encoding = "utf-8"; - } - else if (global_encoding.compare("utf16") == 0) - { - alignment = 2; - byte_length = 2 * tagged_code_units; - encoding = "utf-16-le"; - } - else if (global_encoding.compare("latin1+utf16") == 0) - { - alignment = 2; - if (tagged_code_units & UTF16_TAG) - { - byte_length = 2 * (tagged_code_units ^ UTF16_TAG); - encoding = "utf-16-le"; - } - else - { - byte_length = tagged_code_units; - encoding = "latin-1"; - } - } - - if (ptr != align_to(ptr, alignment)) - { - throw std::runtime_error("Invalid alignment"); - } - if (ptr + byte_length > data.size()) - { - throw std::runtime_error("Out of bounds"); - } - - std::string s; - s.resize(byte_length); - memcpy(&s[0], &data[ptr], byte_length); - return s; -} - -std::string load_string(const wasmtime::Span &data, uint32_t ptr) -{ - uint32_t begin = load_int(data, ptr, 4); - uint32_t tagged_code_units = load_int(data, ptr + 4, 4); - return load_string_from_range(data, begin, tagged_code_units); -} - -// Storing --- -void store_int(const wasmtime::Span &data, int64_t v, size_t ptr, size_t nbytes, bool _signed = false) -{ - // convert v to little-endian byte array - std::vector bytes(nbytes); - for (size_t i = 0; i < nbytes; i++) - { - bytes[i] = (v >> (i * 8)) & 0xFF; - } - // copy bytes to memory - memcpy(&data[ptr], bytes.data(), nbytes); -} - -// Other --- -std::vector read_wasm_binary_to_buffer(const std::string &filename) -{ - std::ifstream file(filename, std::ios::binary | std::ios::ate); - if (!file) - { - throw std::runtime_error("Failed to open file"); - } - - std::streamsize size = file.tellg(); - file.seekg(0, std::ios::beg); - - std::vector buffer(size); - if (!file.read(reinterpret_cast(buffer.data()), size)) - { - throw std::runtime_error("Failed to read file"); - } - - return buffer; -} - -std::string extractContentInDoubleQuotes(const std::string &input) -{ - - int firstQuote = input.find_first_of('"'); - int secondQuote = input.find('"', firstQuote + 1); - if (firstQuote == std::string::npos || secondQuote == std::string::npos) - { - return ""; - } - return input.substr(firstQuote + 1, secondQuote - firstQuote - 1); -} - -std::pair splitQualifiedID(const std::string &qualifiedName) -{ - std::istringstream iss(qualifiedName); - std::vector tokens; - std::string token; - - while (std::getline(iss, token, '.')) - { - tokens.push_back(token); - } - if (tokens.size() != 2) - { - throw std::runtime_error("Invalid import function " + qualifiedName + ", expected format: ."); - } - return std::make_pair(tokens[0], tokens[1]); -} - -std::string createQualifiedID(const std::string &wasmName, const std::string &funcName) -{ - return wasmName + "." + funcName; -} \ No newline at end of file diff --git a/plugins/wasmembed/secure-enclave/secure-enclave.cpp b/plugins/wasmembed/secure-enclave/secure-enclave.cpp deleted file mode 100644 index d8dd157f21c..00000000000 --- a/plugins/wasmembed/secure-enclave/secure-enclave.cpp +++ /dev/null @@ -1,502 +0,0 @@ -#include "secure-enclave.hpp" - -#include "abi.hpp" - -#include -#include - -std::shared_ptr embedContextCallbacks; - -#define NENABLE_TRACE - -#ifdef ENABLE_TRACE -#define TRACE(format, ...) embedContextCallbacks->DBGLOG(format, ##__VA_ARGS__) -#else -#define TRACE(format, ...) \ - do \ - { \ - } while (0) -#endif - -class WasmEngine -{ -protected: - wasmtime::Engine engine; - - std::map wasmInstances; - std::map wasmMems; - std::map wasmFuncs; - -public: - wasmtime::Store store; - - WasmEngine() : store(engine) - { - } - - ~WasmEngine() - { - } - - bool hasInstance(const std::string &wasmName) - { - return wasmInstances.find(wasmName) != wasmInstances.end(); - } - - wasmtime::Instance getInstance(const std::string &wasmName) - { - auto instanceItr = wasmInstances.find(wasmName); - if (instanceItr == wasmInstances.end()) - throw std::runtime_error("Wasm instance not found: " + wasmName); - return instanceItr->second; - } - - void registerInstance(const std::string &wasmName, const std::variant> &wasm) - { - TRACE("registerInstance %s", wasmName.c_str()); - auto instanceItr = wasmInstances.find(wasmName); - if (instanceItr == wasmInstances.end()) - { - TRACE("resolveModule %s", wasmName.c_str()); - auto module = std::holds_alternative(wasm) ? wasmtime::Module::compile(engine, std::get(wasm)).unwrap() : wasmtime::Module::compile(engine, std::get>(wasm)).unwrap(); - TRACE("resolveModule2 %s", wasmName.c_str()); - - wasmtime::WasiConfig wasi; - wasi.inherit_argv(); - wasi.inherit_env(); - wasi.inherit_stdin(); - wasi.inherit_stdout(); - wasi.inherit_stderr(); - store.context().set_wasi(std::move(wasi)).unwrap(); - TRACE("resolveModule3 %s", wasmName.c_str()); - - wasmtime::Linker linker(engine); - linker.define_wasi().unwrap(); - TRACE("resolveModule4 %s", wasmName.c_str()); - - auto callback = [this, wasmName](wasmtime::Caller caller, uint32_t msg, uint32_t msg_len) - { - TRACE("callback: %i %i", msg_len, msg); - - auto data = this->getData(wasmName); - auto msg_ptr = (char *)&data[msg]; - std::string str(msg_ptr, msg_len); - embedContextCallbacks->DBGLOG("from wasm: %s", str.c_str()); - }; - auto host_func = linker.func_wrap("$root", "dbglog", callback).unwrap(); - - auto newInstance = linker.instantiate(store, module).unwrap(); - linker.define_instance(store, "linking2", newInstance).unwrap(); - - TRACE("resolveModule5 %s", wasmName.c_str()); - - wasmInstances.insert(std::make_pair(wasmName, newInstance)); - - for (auto exportItem : module.exports()) - { - auto externType = wasmtime::ExternType::from_export(exportItem); - std::string name(exportItem.name()); - if (std::holds_alternative(externType)) - { - TRACE("Exported function: %s", name.c_str()); - auto func = std::get(*newInstance.get(store, name)); - wasmFuncs.insert(std::make_pair(wasmName + "." + name, func)); - } - else if (std::holds_alternative(externType)) - { - TRACE("Exported memory: %s", name.c_str()); - auto memory = std::get(*newInstance.get(store, name)); - wasmMems.insert(std::make_pair(wasmName + "." + name, memory)); - } - else if (std::holds_alternative(externType)) - { - TRACE("Exported table: %s", name.c_str()); - } - else if (std::holds_alternative(externType)) - { - TRACE("Exported global: %s", name.c_str()); - } - else - { - TRACE("Unknown export type"); - } - } - } - } - - bool hasFunc(const std::string &qualifiedID) - { - return wasmFuncs.find(qualifiedID) != wasmFuncs.end(); - } - - wasmtime::Func getFunc(const std::string &qualifiedID) - { - auto found = wasmFuncs.find(qualifiedID); - if (found == wasmFuncs.end()) - throw std::runtime_error("Wasm function not found: " + qualifiedID); - return found->second; - } - - wasmtime::ValType::ListRef getFuncParams(const std::string &qualifiedID) - { - auto func = getFunc(qualifiedID); - wasmtime::FuncType funcType = func.type(store.context()); - return funcType->params(); - } - - wasmtime::ValType::ListRef getFuncResults(const std::string &qualifiedID) - { - auto func = getFunc(qualifiedID); - wasmtime::FuncType funcType = func.type(store.context()); - return funcType->results(); - } - - std::vector call(const std::string &qualifiedID, const std::vector ¶ms) - { - return getFunc(qualifiedID).call(store, params).unwrap(); - } - - std::vector callRealloc(const std::string &wasmName, const std::vector ¶ms) - { - return call(createQualifiedID(wasmName, "cabi_realloc"), params); - } - - wasmtime::Span getData(const std::string &wasmName) - { - auto found = wasmMems.find(createQualifiedID(wasmName, "memory")); - if (found == wasmMems.end()) - throw std::runtime_error("Wasm memory not found: " + wasmName); - return found->second.data(store.context()); - } -}; -std::unique_ptr wasmEngine; - -class SecureFunction : public ISecureEnclave -{ - std::string wasmName; - std::string funcName; - std::string qualifiedID; - - const IThorActivityContext *activityCtx = nullptr; - std::vector args; - std::vector results; - -public: - SecureFunction() - { - TRACE("se:constructor"); - } - - virtual ~SecureFunction() override - { - TRACE("se:destructor"); - - // Garbage Collection --- - // Function results --- - auto gc_func_name = createQualifiedID(wasmName, "cabi_post_" + funcName); - if (wasmEngine->hasFunc(gc_func_name)) - { - auto func = wasmEngine->getFunc(gc_func_name); - for (auto &result : results) - { - func.call(wasmEngine->store, {result}).unwrap(); - } - } - } - - // IEmbedFunctionContext --- - void setActivityContext(const IThorActivityContext *_activityCtx) - { - activityCtx = _activityCtx; - } - - virtual void Link() const - { - } - - virtual bool Release() const - { - return false; - }; - - virtual IInterface *bindParamWriter(IInterface *esdl, const char *esdlservice, const char *esdltype, const char *name) - { - TRACE("paramWriterCommit"); - return NULL; - } - virtual void paramWriterCommit(IInterface *writer) - { - TRACE("paramWriterCommit"); - } - virtual void writeResult(IInterface *esdl, const char *esdlservice, const char *esdltype, IInterface *writer) - { - TRACE("writeResult"); - } - virtual void bindBooleanParam(const char *name, bool val) - { - TRACE("bindBooleanParam %s %i", name, val); - args.push_back(val); - } - virtual void bindDataParam(const char *name, size32_t len, const void *val) - { - TRACE("bindDataParam %s %d", name, len); - } - virtual void bindFloatParam(const char *name, float val) - { - TRACE("bindFloatParam %s %f", name, val); - args.push_back(val); - } - virtual void bindRealParam(const char *name, double val) - { - TRACE("bindRealParam %s %f", name, val); - args.push_back(val); - } - virtual void bindSignedSizeParam(const char *name, int size, __int64 val) - { - TRACE("bindSignedSizeParam %s %i %lld", name, size, val); - if (size <= 4) - args.push_back(static_cast(val)); - else - args.push_back(static_cast(val)); - } - virtual void bindSignedParam(const char *name, __int64 val) - { - TRACE("bindSignedParam %s %lld", name, val); - args.push_back(static_cast(val)); - } - virtual void bindUnsignedSizeParam(const char *name, int size, unsigned __int64 val) - { - TRACE("bindUnsignedSizeParam %s %i %llu", name, size, val); - if (size <= 4) - args.push_back(static_cast(val)); - else - args.push_back(static_cast(val)); - } - virtual void bindUnsignedParam(const char *name, unsigned __int64 val) - { - TRACE("bindUnsignedParam %s %llu", name, val); - args.push_back(static_cast(val)); - } - virtual void bindStringParam(const char *paramName, size32_t len, const char *val) - { - TRACE("bindStringParam %s %d %s", paramName, len, val); - auto memIdxVar = wasmEngine->callRealloc(wasmName, {0, 0, 1, (int32_t)len}); - auto memIdx = memIdxVar[0].i32(); - auto mem = wasmEngine->getData(wasmName); - for (int i = 0; i < len; i++) - { - mem[memIdx + i] = val[i]; - } - args.push_back(memIdx); - args.push_back((int32_t)len); - } - virtual void bindVStringParam(const char *name, const char *val) - { - TRACE("bindVStringParam %s %s", name, val); - auto len = strlen(val); - auto memIdxVar = wasmEngine->callRealloc(wasmName, {0, 0, 1, (int32_t)len}); - auto memIdx = memIdxVar[0].i32(); - auto mem = wasmEngine->getData(wasmName); - for (int i = 0; i < len; i++) - { - mem[memIdx + i] = val[i]; - } - args.push_back(memIdx); - args.push_back((int32_t)len); - } - virtual void bindUTF8Param(const char *name, size32_t chars, const char *val) - { - TRACE("bindUTF8Param %s %d %s", name, chars, val); - auto memIdxVar = wasmEngine->callRealloc(wasmName, {0, 0, 1, (int32_t)chars}); - auto memIdx = memIdxVar[0].i32(); - auto mem = wasmEngine->getData(wasmName); - for (int i = 0; i < chars; i++) - { - mem[memIdx + i] = val[i]; - } - args.push_back(memIdx); - args.push_back((int32_t)chars); - } - virtual void bindUnicodeParam(const char *name, size32_t chars, const UChar *val) - { - TRACE("bindUnicodeParam %s %d %S", name, chars, reinterpret_cast(val)); - auto memIdxVar = wasmEngine->callRealloc(wasmName, {0, 0, 2, (int32_t)chars * 2}); - auto memIdx = memIdxVar[0].i32(); - auto mem = wasmEngine->getData(wasmName); - for (int i = 0; i < chars * 2; i += 2) - { - mem[memIdx + i] = val[i]; - } - args.push_back(memIdx); - args.push_back((int32_t)chars); - } - virtual void bindSetParam(const char *name, int elemType, size32_t elemSize, bool isAll, size32_t totalBytes, const void *setData) - { - TRACE("bindSetParam %s %d %d %d %d %p", name, elemType, elemSize, isAll, totalBytes, setData); - } - virtual void bindRowParam(const char *name, IOutputMetaData &metaVal, const byte *val) override - { - TRACE("bindRowParam %s %p", name, val); - } - virtual void bindDatasetParam(const char *name, IOutputMetaData &metaVal, IRowStream *val) - { - TRACE("bindDatasetParam %s %p", name, val); - } - virtual bool getBooleanResult() - { - TRACE("getBooleanResult"); - return results[0].i32(); - } - virtual void getDataResult(size32_t &__len, void *&__result) - { - TRACE("getDataResult"); - } - virtual double getRealResult() - { - TRACE("getRealResult"); - if (results[0].kind() == wasmtime::ValKind::F64) - return (int32_t)results[0].f64(); - return results[0].f32(); - } - virtual __int64 getSignedResult() - { - TRACE("getSignedResult"); - if (results[0].kind() == wasmtime::ValKind::I64) - return (int32_t)results[0].i64(); - return results[0].i32(); - } - virtual unsigned __int64 getUnsignedResult() - { - TRACE("getUnsignedResult"); - if (results[0].kind() == wasmtime::ValKind::I64) - return (int32_t)results[0].i64(); - return results[0].i32(); - } - virtual void getStringResult(size32_t &__chars, char *&__result) - { - TRACE("getStringResult %zu", results.size()); - auto ptr = results[0].i32(); - auto data = wasmEngine->getData(wasmName); - - uint32_t begin = load_int(data, ptr, 4); - TRACE("begin %u", begin); - uint32_t tagged_code_units = load_int(data, ptr + 4, 4); - TRACE("tagged_code_units %u", tagged_code_units); - std::string s = load_string(data, ptr); - TRACE("load_string %s", s.c_str()); - __chars = s.length(); - __result = reinterpret_cast(embedContextCallbacks->rtlMalloc(__chars)); - s.copy(__result, __chars); - } - virtual void getUTF8Result(size32_t &__chars, char *&__result) - { - TRACE("getUTF8Result"); - } - virtual void getUnicodeResult(size32_t &__chars, UChar *&__result) - { - TRACE("getUnicodeResult"); - } - virtual void getSetResult(bool &__isAllResult, size32_t &__resultBytes, void *&__result, int elemType, size32_t elemSize) - { - TRACE("getSetResult"); - } - virtual IRowStream *getDatasetResult(IEngineRowAllocator *_resultAllocator) - { - TRACE("getDatasetResult"); - return NULL; - } - virtual byte *getRowResult(IEngineRowAllocator *_resultAllocator) - { - TRACE("getRowResult"); - return NULL; - } - virtual size32_t getTransformResult(ARowBuilder &builder) - { - TRACE("getTransformResult"); - return 0; - } - virtual void loadCompiledScript(size32_t chars, const void *_script) override - { - TRACE("loadCompiledScript %p", _script); - } - virtual void enter() override - { - TRACE("enter"); - } - virtual void reenter(ICodeContext *codeCtx) override - { - TRACE("reenter"); - } - virtual void exit() override - { - TRACE("exit"); - } - virtual void compileEmbeddedScript(size32_t lenChars, const char *_utf) override - { - TRACE("compileEmbeddedScript"); - std::string utf(_utf, lenChars); - funcName = extractContentInDoubleQuotes(utf); - wasmName = "embed_" + funcName; - qualifiedID = createQualifiedID(wasmName, funcName); - wasmEngine->registerInstance(wasmName, utf); - } - virtual void importFunction(size32_t lenChars, const char *qualifiedName) override - { - TRACE("importFunction: %s", qualifiedName); - - qualifiedID = std::string(qualifiedName, lenChars); - auto [_wasmName, _funcName] = splitQualifiedID(qualifiedID); - wasmName = _wasmName; - funcName = _funcName; - - if (!wasmEngine->hasInstance(wasmName)) - { - std::string fullPath = embedContextCallbacks->resolveManifestPath((wasmName + ".wasm").c_str()); - auto wasmFile = read_wasm_binary_to_buffer(fullPath); - wasmEngine->registerInstance(wasmName, wasmFile); - } - } - virtual void callFunction() - { - TRACE("callFunction %s", qualifiedID.c_str()); - results = wasmEngine->call(qualifiedID, args); - } -}; - -SECUREENCLAVE_API void init(std::shared_ptr embedContext) -{ - embedContextCallbacks = embedContext; - wasmEngine = std::make_unique(); - TRACE("init"); -} - -SECUREENCLAVE_API void kill() -{ - TRACE("kill"); - wasmEngine.reset(); - embedContextCallbacks.reset(); -} - -SECUREENCLAVE_API std::unique_ptr createISecureEnclave() -{ - return std::make_unique(); -} - -SECUREENCLAVE_API void syntaxCheck(size32_t &__lenResult, char *&__result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions) -{ - std::string errMsg = ""; - try - { - wasmtime::Engine engine; - wasmtime::Store store(engine); - auto module = wasmtime::Module::compile(engine, body); - } - catch (const wasmtime::Error &e) - { - errMsg = e.message(); - } - - __lenResult = errMsg.length(); - __result = reinterpret_cast(embedContextCallbacks->rtlMalloc(__lenResult)); - errMsg.copy(__result, __lenResult); -} diff --git a/plugins/wasmembed/secure-enclave/secure-enclave.hpp b/plugins/wasmembed/secure-enclave/secure-enclave.hpp deleted file mode 100644 index 82e8c800fb2..00000000000 --- a/plugins/wasmembed/secure-enclave/secure-enclave.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "platform.h" -#include "eclrtl.hpp" - -#ifdef SECUREENCLAVE_EXPORTS - #define SECUREENCLAVE_API DECL_EXPORT -#else - #define SECUREENCLAVE_API DECL_IMPORT -#endif - -#include - -interface IWasmEmbedCallback -{ - virtual inline void DBGLOG(char const *format, ...) __attribute__((format(printf, 2, 3))) = 0; - virtual void *rtlMalloc(size32_t size) = 0; - - virtual const char *resolveManifestPath(const char *leafName) = 0; -}; - -interface ISecureEnclave : extends IEmbedFunctionContext -{ - virtual ~ISecureEnclave() = default; -}; - -SECUREENCLAVE_API void init(std::shared_ptr embedContext); -SECUREENCLAVE_API void kill(); -SECUREENCLAVE_API std::unique_ptr createISecureEnclave(); -SECUREENCLAVE_API void syntaxCheck(size32_t &__lenResult, char *&__result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions); diff --git a/plugins/wasmembed/util.cpp b/plugins/wasmembed/util.cpp new file mode 100644 index 00000000000..32d929c5b74 --- /dev/null +++ b/plugins/wasmembed/util.cpp @@ -0,0 +1,59 @@ +#include "util.hpp" +#include + +#include +#include + +std::vector read_wasm_binary_to_buffer(const std::string &filename) +{ + std::ifstream file(filename, std::ios::binary | std::ios::ate); + if (!file) + { + throw makeStringException(0, "Failed to open file"); + } + + std::streamsize size = file.tellg(); + file.seekg(0, std::ios::beg); + + std::vector buffer(size); + if (!file.read(reinterpret_cast(buffer.data()), size)) + { + throw makeStringException(1, "Failed to read file"); + } + + return buffer; +} + +std::string extractContentInDoubleQuotes(const std::string &input) +{ + + auto firstQuote = input.find_first_of('"'); + auto secondQuote = input.find('"', firstQuote + 1); + if (firstQuote == std::string::npos || secondQuote == std::string::npos) + { + return ""; + } + return input.substr(firstQuote + 1, secondQuote - firstQuote - 1); +} + +std::pair splitQualifiedID(const std::string &qualifiedName) +{ + std::istringstream iss(qualifiedName); + std::vector tokens; + std::string token; + + while (std::getline(iss, token, '.')) + { + tokens.push_back(token); + } + if (tokens.size() != 2) + { + throw makeStringExceptionV(3, "Invalid import function %s, expected format: .", qualifiedName.c_str()); + } + return std::make_pair(tokens[0], tokens[1]); +} + +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName) +{ + return wasmName + "." + funcName; +} \ No newline at end of file diff --git a/plugins/wasmembed/util.hpp b/plugins/wasmembed/util.hpp new file mode 100644 index 00000000000..e9c5e9e0f47 --- /dev/null +++ b/plugins/wasmembed/util.hpp @@ -0,0 +1,7 @@ +#include +#include + +std::vector read_wasm_binary_to_buffer(const std::string &filename); +std::string extractContentInDoubleQuotes(const std::string &input); +std::pair splitQualifiedID(const std::string &qualifiedName); +std::string createQualifiedID(const std::string &wasmName, const std::string &funcName); diff --git a/plugins/wasmembed/wasmembed.cpp b/plugins/wasmembed/wasmembed.cpp index 717e4958d95..c27714d69aa 100644 --- a/plugins/wasmembed/wasmembed.cpp +++ b/plugins/wasmembed/wasmembed.cpp @@ -1,9 +1,10 @@ #include "platform.h" #include "hqlplugins.hpp" +#include "rtlconst.hpp" #include "rtlfield.hpp" #include "enginecontext.hpp" -#include "secure-enclave/secure-enclave.hpp" +#include "secure-enclave.hpp" static const char *compatibleVersions[] = { "WASM Embed Helper 1.0.0", @@ -31,94 +32,35 @@ extern "C" DECL_EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb) namespace wasmLanguageHelper { - class Callbacks : public IWasmEmbedCallback + class WasmEmbedContext : public CInterfaceOf { - protected: - bool manifestAdded = false; - StringArray manifestModules; - public: - Callbacks() - { - } - ~Callbacks() + virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options) override { + return createFunctionContextEx(nullptr, nullptr, flags, options); } - void manifestPaths(ICodeContext *codeCtx) + virtual IEmbedFunctionContext *createFunctionContextEx(ICodeContext *codeCtx, const IThorActivityContext *activityContext, unsigned flags, const char *options) override { - if (codeCtx && !manifestAdded) + StringArray manifestModules; + if (codeCtx) { - manifestAdded = true; - IEngineContext *engine = codeCtx->queryEngineContext(); + auto engine = codeCtx->queryEngineContext(); if (engine) - { engine->getManifestFiles("wasm", manifestModules); - } } + return createISecureEnclave(manifestModules); } - // IWasmEmbedCallback --- - virtual inline void DBGLOG(char const *format, ...) override - { - va_list args; - va_start(args, format); - VALOG(MCdebugInfo, unknownJob, format, args); - va_end(args); - } - - virtual void *rtlMalloc(size32_t size) override - { - return ::rtlMalloc(size); - } - - virtual const char *resolveManifestPath(const char *leafName) override - { - if (leafName && *leafName) - { - ForEachItemIn(idx, manifestModules) - { - const char *path = manifestModules.item(idx); - if (endsWith(path, leafName)) - return path; - } - } - return nullptr; - } - }; - std::shared_ptr callbacks; - - class WasmEmbedContext : public CInterfaceOf - { - std::unique_ptr enclave; - - public: - WasmEmbedContext() - { - enclave = createISecureEnclave(); - } - virtual ~WasmEmbedContext() override - { - } - // IEmbedContext --- - virtual IEmbedFunctionContext *createFunctionContext(unsigned flags, const char *options) override - { - return createFunctionContextEx(nullptr, nullptr, flags, options); - } - virtual IEmbedFunctionContext *createFunctionContextEx(ICodeContext *ctx, const IThorActivityContext *activityContext, unsigned flags, const char *options) override - { - callbacks->manifestPaths(ctx); - return enclave.get(); - } virtual IEmbedServiceContext *createServiceContext(const char *service, unsigned flags, const char *options) override { throwUnexpected(); return nullptr; } - }; + } theEmbedContext; extern DECL_EXPORT IEmbedContext *getEmbedContext() { - return new WasmEmbedContext(); + return LINK(&theEmbedContext); } extern DECL_EXPORT void syntaxCheck(size32_t &__lenResult, char *&__result, const char *funcname, size32_t charsBody, const char *body, const char *argNames, const char *compilerOptions, const char *persistOptions) @@ -130,16 +72,3 @@ namespace wasmLanguageHelper } } // namespace - -MODULE_INIT(INIT_PRIORITY_STANDARD) -{ - wasmLanguageHelper::callbacks = std::make_shared(); - init(wasmLanguageHelper::callbacks); - return true; -} - -MODULE_EXIT() -{ - kill(); - wasmLanguageHelper::callbacks.reset(); -} \ No newline at end of file diff --git a/system/jlib/jarray.hpp b/system/jlib/jarray.hpp index a2c1240bb6d..9870c8d9c31 100644 --- a/system/jlib/jarray.hpp +++ b/system/jlib/jarray.hpp @@ -153,7 +153,7 @@ class ArrayOf : public AllocatorOf typedef int (*CompareFunc)(const MEMBER *, const MEMBER *); // Should really be const, as should the original array functions public: - ~ArrayOf() { kill(); } + ~ArrayOf() { kill(); } MEMBER & operator[](size_t pos) { return element((aindex_t)pos); } const MEMBER & operator[](size_t pos) const { return element((aindex_t)pos); } diff --git a/system/jlib/jbuff.hpp b/system/jlib/jbuff.hpp index 5d6392dbec6..010e9dcc66d 100644 --- a/system/jlib/jbuff.hpp +++ b/system/jlib/jbuff.hpp @@ -66,10 +66,10 @@ template class OwnedPtrCustomFree CLASS *ptr = nullptr; public: - OwnedPtrCustomFree() { } - OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } - OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } - ~OwnedPtrCustomFree() { safeFree(ptr); } + OwnedPtrCustomFree() { } + OwnedPtrCustomFree(CLASS *_ptr) : ptr(_ptr) { } + OwnedPtrCustomFree(SELF &&_ptr) { ptr = _ptr.getClear(); } + ~OwnedPtrCustomFree() { safeFree(ptr); } void operator = (CLASS * _ptr) { diff --git a/system/jlib/jhash.hpp b/system/jlib/jhash.hpp index c8790fe2e4c..3653abd4c21 100644 --- a/system/jlib/jhash.hpp +++ b/system/jlib/jhash.hpp @@ -468,13 +468,13 @@ template class CMinHashTable } public: - CMinHashTable(unsigned _initialSize = 7) + CMinHashTable(unsigned _initialSize = 7) { htn = _initialSize; n = 0; table = (C **)calloc(sizeof(C *),htn); } - ~CMinHashTable() + ~CMinHashTable() { C **t = table+htn; while (t--!=table) @@ -625,7 +625,7 @@ template class CTimeLimitedCache { public: - CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) + CTimeLimitedCache(unsigned timeoutMs=defaultCacheTimeoutMs) { timeoutPeriodCycles = ((cycle_t)timeoutMs) * queryOneSecCycles() / 1000; } diff --git a/system/jlib/jiface.hpp b/system/jlib/jiface.hpp index 4f3d6b2a744..ca502ab010d 100644 --- a/system/jlib/jiface.hpp +++ b/system/jlib/jiface.hpp @@ -88,8 +88,8 @@ class CSimpleInterfaceOf : public INTERFACE } private: - CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; - CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; + CSimpleInterfaceOf(const CSimpleInterfaceOf&) = delete; + CSimpleInterfaceOf(CSimpleInterfaceOf &&) = delete; CSimpleInterfaceOf & operator = (const CSimpleInterfaceOf &) = delete; mutable std::atomic xxcount; }; diff --git a/system/jlib/jsuperhash.hpp b/system/jlib/jsuperhash.hpp index 9936e86112f..db80504cdc2 100644 --- a/system/jlib/jsuperhash.hpp +++ b/system/jlib/jsuperhash.hpp @@ -178,9 +178,9 @@ class SimpleHashTableOf : public SuperHashTableOf { typedef SimpleHashTableOf SELF; public: - SimpleHashTableOf(void) : SuperHashTableOf() { } - SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~SimpleHashTableOf() { SELF::_releaseAll(); } + SimpleHashTableOf(void) : SuperHashTableOf() { } + SimpleHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~SimpleHashTableOf() { SELF::_releaseAll(); } IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(ET, FP); @@ -209,9 +209,9 @@ class OwningSimpleHashTableOf : public SimpleHashTableOf { typedef OwningSimpleHashTableOf SELF; public: - OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } - OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } - ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } + OwningSimpleHashTableOf(void) : SimpleHashTableOf() { } + OwningSimpleHashTableOf(unsigned initsize) : SimpleHashTableOf(initsize) { } + ~OwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -288,9 +288,9 @@ class StringSuperHashTableOf : public SuperHashTableOf { typedef StringSuperHashTableOf SELF; public: - StringSuperHashTableOf(void) : SuperHashTableOf() { } - StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } - ~StringSuperHashTableOf() { SELF::_releaseAll(); } + StringSuperHashTableOf(void) : SuperHashTableOf() { } + StringSuperHashTableOf(unsigned initsize) : SuperHashTableOf(initsize) { } + ~StringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onAdd(void *et __attribute__((unused))) { } virtual void onRemove(void *et __attribute__((unused))) { } @@ -318,9 +318,9 @@ class OwningStringSuperHashTableOf : public StringSuperHashTableOf { typedef OwningStringSuperHashTableOf SELF; public: - OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } - OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } - ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } + OwningStringSuperHashTableOf(void) : StringSuperHashTableOf() { } + OwningStringSuperHashTableOf(unsigned initsize) : StringSuperHashTableOf(initsize) { } + ~OwningStringSuperHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; @@ -422,7 +422,7 @@ class ThreadSafeOwningSimpleHashTableOf : public ThreadSafeSimpleHashTableOf SELF; public: - ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } + ~ThreadSafeOwningSimpleHashTableOf() { SELF::_releaseAll(); } virtual void onRemove(void *et) { ((ET *)et)->Release(); } }; diff --git a/testing/regress/ecl/key/wasmembed.xml b/testing/regress/ecl/key/wasmembed.xml index 5f862a9d026..4400c029084 100644 --- a/testing/regress/ecl/key/wasmembed.xml +++ b/testing/regress/ecl/key/wasmembed.xml @@ -49,3 +49,54 @@ true + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + + + true + diff --git a/testing/regress/ecl/wasmembed.ecl b/testing/regress/ecl/wasmembed.ecl index 143b04598aa..0d16060c1dc 100644 --- a/testing/regress/ecl/wasmembed.ecl +++ b/testing/regress/ecl/wasmembed.ecl @@ -12,23 +12,76 @@ integer2 s16Test (integer2 a, integer2 b) := IMPORT(wasm, 'wasmembed.s16-test'); integer4 s32Test (integer4 a, integer4 b) := IMPORT(wasm, 'wasmembed.s32-test'); integer8 s64Test (integer8 a, integer8 b) := IMPORT(wasm, 'wasmembed.s64-test'); string stringTest (string a, string b) := IMPORT(wasm, 'wasmembed.string-test'); -string7 string5Test (string5 a, string5 b) := IMPORT(wasm, 'wasmembed.string-test'); +string12 string5Test (string5 a, string5 b) := IMPORT(wasm, 'wasmembed.string-test'); varstring varstringTest (varstring a, varstring b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode12 unicode5Test (unicode5 a, unicode5 b) := IMPORT(wasm, 'wasmembed.string-test'); +unicode unicodeTest (unicode a, unicode b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8_12 utf8_5Test (utf8_5 a, utf8_5 b) := IMPORT(wasm, 'wasmembed.string-test'); +utf8 utf8Test (utf8 a, utf8 b) := IMPORT(wasm, 'wasmembed.string-test'); +// '--- bool ---'; boolTest(false, false) = (false AND false); boolTest(false, true) = (false AND true); boolTest(true, false) = (true AND false); boolTest(true, true) = (true AND true); -float32Test(1.0, 2.0) = (1.0 + 2.0); -float64Test(1.0, 2.0) = (1.0 + 2.0); -u8Test(1, 2) = (1 + 2); -u16Test(1, 2) = (1 + 2); -u32Test(1, 2) = (1 + 2); -u64Test(1, 2) = (1 + 2); -s8Test(1, 2) = (1 + 2); -s16Test(1, 2) = (1 + 2); -s32Test(1, 2) = (1 + 2); -s64Test(1, 2) = (1 + 2); -stringTest('aaa', 'bbbb') = ('aaa' + 'bbbb'); -string5Test('aaa', 'bbbb') = (string7)((string5)'aaa' + (string5)'bbbb'); -varstringTest('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb') = ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'); +// '--- float ---'; +ROUND(float32Test((real4)1234.1234, (real4)2345.2345), 3) = ROUND((real4)((real4)1234.1234 + (real4)2345.2345), 3); +float64Test(123456789.123456789, 23456789.23456789) = (real8)((real8)123456789.123456789 + (real8)23456789.23456789); +// '--- unsigned ---'; +u8Test(1, 2) = (unsigned1)(1 + 2); +u8Test(255, 1) = (unsigned1)(255 + 1); +u16Test(1, 2) = (unsigned2)(1 + 2); +u16Test(65535, 1) = (unsigned2)(65535 + 1); +u32Test(1, 2) = (unsigned4)(1 + 2); +u32Test(4294967295, 1) = (unsigned4)(4294967295 + 1); +u64Test(1, 2) = (unsigned8)(1 + 2); +u64Test(18446744073709551615, 1) = (unsigned8)(18446744073709551615 + 1); +// '--- signed ---'; +s8Test(1, 2) = (integer1)(1 + 2); +s8Test(127, 1) = (integer1)(127 + 1); +s8Test(-128, -1) = (integer1)(-128 - 1); + +s16Test(1, 2) = (integer2)(1 + 2); +s16Test(32767, 1) = (integer2)(32767 + 1); +s16Test(-32768, -1) = (integer2)(-32768 - 1); + +s32Test(1, 2) = (integer4)(1 + 2); +s32Test(2147483647, 1) = (integer4)(2147483647 + 1); +s32Test(-2147483648, -1) = (integer4)(-2147483648 - 1); + +s64Test(1, 2) = (integer8)(1 + 2); +s64Test(9223372036854775807, 1) = (integer8)(9223372036854775807 + 1); +s64Test(-9223372036854775808, -1) = (integer8)(-9223372036854775808 - 1); +// '--- string ---'; +varstringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +stringTest('1234567890', 'abcdefghij') = '1234567890' + 'abcdefghij'; +unicodeTest(U'1234567890您好1231231230', U'abcdefghij欢迎光临abcdefghij') = U'1234567890您好1231231230' + U'abcdefghij欢迎光临abcdefghij'; +utf8Test(U8'您好', U8'欢迎光临') = U8'您好' + U8'欢迎光临'; +// '--- string (fixed length) ---'; +string5Test('1234567890', 'abcdefghij') = (string12)((string5)'1234567890' + (string5)'abcdefghij'); +utf8_5Test(U8'您好1234567890', U8'欢迎光临abcdefghij') = (utf8_12)((utf8_5)U8'您好1234567890' + (utf8_5)U8'欢迎光临abcdefghij'); +unicode5Test(U'您好1234567890', U'欢迎光临abcdefghij') = (unicode12)((unicode5)U'您好1234567890' + (unicode5)U'欢迎光临abcdefghij'); +// '--- reentry ---'; +r := RECORD + unsigned1 kind; + string20 word; + unsigned8 doc; + unsigned1 segment; + unsigned8 wpos; + END; +d := dataset('~regress::multi::searchsource', r, THOR); + +r2 := RECORD(r) + boolean passed; +END; + +r2 t(r L) := TRANSFORM + boolean a := u64Test(L.doc, L.wpos) = (unsigned8)(L.doc + L.wpos); + boolean b := stringTest(L.word, L.word) = L.word + L.word; + SELF.passed := a and B; + SELF := L; +END; + +d2 := project(choosen(d, 100), t(LEFT)); +count(d2(passed=false)) = 0; +// '--- --- ---'; diff --git a/testing/regress/ecl/wasmembed.wasm b/testing/regress/ecl/wasmembed.wasm index bb27b76cb859cda62519d509086d10ed6dffc8b9..f2f32c33c958c4eba0a09a7f0c7317aa0b165c2c 100644 GIT binary patch delta 6454 zcmZ`e2|yHAmi2!1MR!9ND9F`P)gXrmf@0%=LVJ_MsPWdA76BD#&TcgtoyJ6yXo8B* z7>_6-o{R?(?-Pu-5j5G!CbOf_$z+q6*-gx5j-5$nocF5%+hn)Z|KEN6v%a6yi*MJ9 z>#eo6{=jM{#*6dBFifintF1FKA7s+cD{#6C3rhrz%JCWqUtCq>DHOEgHMjygy+Nnb z1@M1Hli92_naqA9txlw0Aa6v`WD?DUlNcCHgoyMv$SjIP17aI>U>0=%CX6j z5Gc^6qSR=i0yHE90@E453yz)ZG$#EI5UeXLEG@5E!i3QLytzeA{2YS4ryyKKD$3m+ zr>n^ANts(-?n?0#x;?^aFsr<(!aSF&JYP5i0Z*7`!934Zp6BV`&nJAY1Gi`R@bjQw zIM4@P0K>v`yAQrdaHmQO7`zKRAB;7^n4qrbWrKqV!v8F6{Mp@pv>%yVi_)EiUYa#!NLd%dtkBR)%%BS z=m=Zyi!h5iy`6ClLst?xCYjiYyo$W;vg%&eb`()t%z2a{uZb^YS=YhS`3mfyL=xHC zZ}Aa_2^6;7hmek;n5ywBwfdqXyqM#h=%{&vA@+i3w*5=|{tOiP$eOGZp4 ztD2%iQ%rNi0b|J&pC!^xb=>|BZZChf!j(ntpCXWUaUB~A6r{a)PuC;sILE1!rhx!d zWf>%uVpGdY6{9~?YiWZz$B0M?QPp|5rDjaihueKY$bvdaO(EpPA&lWmXN>e3S2CHo zV~|$*Bdf4h-@W*Cl@!KR9zPZ};Nw5AJfm7!IlW%oq% zZGWT&Er!vs7N;8GhL5K_J2qvh#KwHHjVEPiX6mRGGz<9hWYRirgAZYnNGotXU)IEu z&}GPm{dh6J8ghV3&;&9lbSlsuEj8jd0r3a{={^CxVF<#P0_JpxAN;BKAre0dO;~T7 zZhD_H1(R%&_MpaeFyyEzoF=F7%{g68r+Qy9^@C&hXHzO1$2fDZ%oALM6i%wWk#+=S zILncT^T8&}tc;LHNPAT~BRL^a86}URfv+}qmrrqWq=}O=l(D2Wwo5Bl&LyE^<|H_U z_swHrFD3-`p>}d$8c|jS=1>q<1LNaPa}%lsvr-|O2IzN7K;Jk|M?mHa*7r*Lzmi_&S_<0gf#`N@Ts#0b8$mkz=47@nx-K+%pg0RRzvPJp_ z-c8SA|E$K0e&gBWYFyQCI=qMX`{lwGygNDq9sSc_1zzkQjcW;dm7t9g8~JegmWrLp z+?q^_L?<<&E2DQv=l-ISgXLi90`AXv7SLNl=S73EqI;4EY-%ew|fJ7VZu0NfvfyTXEgcaRhbZGjZy$ zVB#+r7K1+wbrHu0&BC&5POr|6!8eBaur(c6Pt463PrNNTGS+4Lur*y+p`%n?!B%|a z$bok#jeW2QKN~(Bc3{5|>F_QVjA($J>gXvWH^DYMHIk^!e;7FiU=Q{iJsGy+8>4eo zk)BZ3{Al!|09vv0CoHG-%L#GR8Yj-CcELm|wXaW{VA`jSIGs__zUGG$_XRyvvEO|XlYPav0&jhJdo;eN=cfjSyiNiT&izgUvigg%&oc$4VdA6DM3}B zD?FWorWk>=1c$^wN=TWjS@`W9q}_Ph{eseQNwd(IEYWb&dCAXkQz+pwT^wW)!#p;F zdIu4t{W#TQgJ0qj&nRfbi=NeR2y++iq4wK_^WgwaTQr8|Mctw>c)$6?qR)-q6WpJ4 zl((h;?n07If)5Jj5O?cyBZY9>q~cPTAKb$a4)=p2{NPAGILZ$mAMJ-&{otN{aEu=; zbJ#B^#gAu`PBxEw^??@7V(Obi4JgvZ5**fM_nX*B$u7iN%YvT3! zu5x!FB;kabURwT0!HAj+BEC`+#WYs5w1wf}8a-~R3CG(tdZr7<=i0(SkKfirGD8HO zZ;OZyh*(gOpYL?%=ehE#oct5sNuSp$V8ooY5n#fZYa^LC0(Z7Ypl5A&LkBkqgEmCO za|KJx&rI;|YA65hR}D#6+pb{RMk79d+suLoA+*ILJuTu?-x8_{JqxSKLY{g-=!W0b zMrhoHp6&!hLnyhmvam#M18ZsmSdU}^<9MDdoP)_Mg6?RrpwH@d-yMY! zTccT;F9^9I6o)k#Sh`mg*KA9Ge%NPQ3G2Tc?>5fF@3+OW3>QA!GZR~Og}?w@_1+z3 zU#$k$v^||={ub{Xux{9Kl?@!J+Wulkk~pXuuySiR{Bc(tj;_mv!4$x9Hl&)gX5+Uz zwdmd)jO9lHSyC3(?tBe~VrYFX8|Dk)RDA+u*_+IsNmC;pwl5T(#mW0J*>guals2+)W64w$&fK3OK7SK1W!q3Jt~KMw z`&Y2>ys}gA-~k&BuzTPu&FVMLR*T<4{l|X*HG%yu;Ne# zOu;>eY;5ZHM3{m-57)soyl{9vo4y2Z>`%ohN1lZlSbt;>o7v=-Kopzx45r`ehC%OV zvKQx&?PPpqs|B^kWL_IKdpilgjj2r`@Dh$an$Db~NjwzSA5DijcnF@ud1?wPslqK6WALj}lfZ>TPFJzglUQ+KAbxeanw2fU zN2j}^d@ccZpGjfm)m@sFv!htW7`$^%LHU#cn@$?B@l1%OcT!~uRv)wBI5PR*Y;RW8 zX<;&RKkp~5!G?1R^L&iUPbOp5g>XD`G6bW}d)dO}q%Wg%T#p}|pUM_ZBBeoOJAx%G z#_|hi*vp*`a$yO+d+}XZihW!1fL2?}1h%Xar?w8nsMbNS9H+ESV857$!IuW(Mk-`7 z74;vj!Qe&d(g?Adt};g&`>3N-dubfq(>}j6fbM9aAG{>KMju0)ugkdUgYBA?uo3X$ z;W(Uixq{z>F3%8Ory*;*oQcjWN#Yxju8!D_D?RB}bpFZ+x_ZFXY`UsAu4ZqzYQ^nW zV=;&Nd(>A`Z@m_aE!5Ax7Q5jZu?fwg{vP$!)LXCf%aP;yWV#$}y50yI(Q#t|T~r!x z)Y3&I>t+>QQI6aU7dO*x2)80NNn0Qcy|+SH(pG$YvoBp(`raBxm(b<6dW+kkKgAY> zS8j!ScYv-`eaWo@LxIZ&?gY)es=~s0&=t5iunY7Z&%F&0SWx7qy}ZzwzhGfm$!-WP zD)SUpmF2md^pKqADwJ@p1a6NFYWnW ze!?AqpeH1L&NceWsIT$A1g&})9)v)r)9ooJDtD^*A<(J(MTeg-RKgL^JDo0WtO?AW zDjaG_wqRjt z>5`KWtVUbvq%+m!Ebx?9xlhqP)^Ui?85kv=rhSV}l7Oz01aoi8a0ZKSTg9`Gsh(eS zxZx*l7MfCzZ^x_W__wz$*m^sGpXJ!6%_3f)UE6okg~3H!PiO2qZ9UN47E5Pc7#^hF z&>lt~tugJH^l`kt-G-%W^;q3*!Kv-Bcpvq6n&5Q=|FJzxyb6Q5Vobebq5s{;`bms; L+q&b>J6rw*7t^fF delta 6126 zcmZ`-2V7KF)_>=|HwE6nJP?r1n*ph!U_n$A#zxqvQKN~8(UCGDIxqwV+s2F~&90op zu80aE8XK`<0lUVU>}Hc}Hf2jTo0vqueA!LeEZ@0r1|{G3`$oCj&D!|?Fxuyt$wnWRawVMd9)+ToZj z=zJE*Gq<91~5qm08uoP8YoGEK{J`nW?51U23a;sU@%A?kYuW5ib0JfLdoyZk4%O@ zN@UWgSF2$H0ujsP$gC;&);VAB7pk29i6}enaukkdPW;k7i?!JAz z`Wc^|)z7P+1w&1?S3C#Cn#?S(+(vR|lXj9j-Oq!en%qb8ycfV&-RXak z2DxijuYjKk1RHUG01m+s@UQQHi+G!rc$P{adg2$+Pq5t=W?RJsO;$zQ`@$?$NM-^r zBxNo4ut=()Y~T{vuaX_Oor+F7DG%-0vsCWBP75vAD9XweJPXXSLY7xn#Su0mMN$6X ziy1zP>mn7lRj-k=@;h#@0s&SX<;tRTq6#TY7$AnLFiaRgfDS!6Yc{mmgvYG9C{;{m zIVz~SDCO?|-#GbE;}3Lk32*rFe<93LHgcKesbWu|ug&wD0Lqxml9er}=(|U3SxiRR zYwh;XY9wo(KFYKHQPbfMjoHe>MnX?P_Trq_N4WG5mmc9n!+l|vcR1vEz&wDaW|0%4XSwK?2##veLv6LEc5HvU7qckLCyY(y*&6U68GkDsl3oJS6eSpaMsA zs!l_#I`4=owVWNRcS@-m71YKzg;d}aa9!u?3bmpPltyKpCdp@#e7zKHN+&-q&d;no z6}k{aikBWfowTVVcqa_aQjIUOOfFuoL?2f)V^x}+ypp^miCvj{NDH1dgnM>!hGf}7 zgQ1EewSf?o-5j*C0HKpWmC0e8Z3?SZ%5m<&sm})RaDx}I^5IYg8~?{g(8mCvPg1S2d77g{`AQ)KID>Dzro?vMZ+{9zJ8-%4KDCYz;FCAp&3>GnBaZf#bQ*A7TU&SRZ{lj6aG(ojqoO3@-O!} z9{Asm2gxxJUz1nD5ezj=mRq<3trc0>gY!)LgWu8`WT+X$mL*foRE}a+Kp!}UWdXh5 zIBpE+G5Q2|p#~?l+Mjk8%P^vU7^j{zlV!L%TxrtcjNm}BWu!XN2aMuCjwN3mp^j4X zl~d>!7z<699GC*9a6;f?ByS1qN#)hR3_^Yk%%jwU%!vuDJf7^0U#ZG!rE*avaFnvjbW7_Btv(uiRDvV$2PK0{DM3yu*9TR=Wz+>v z47}Xwrd%Nj&B2klI5?Yl{5UbzkBC-vs-JQeKMU>#?f8B0YRmbD4K8Sso^}pBpR%+) zN*051v1NBR6Tl@r95M>-;Hus+7#cbcuH)*^R=9?FQ88E=mIqf`n!+*x-o_8RSHo=_ z8~!wPs4?70Ww(eND#t~$mtlD7>iS*lIYZWIV#=sC#?rGoy>AN zW?I9sGA+R1TX1NJ&4f?1d;_Cr!UfzJoeZDi+tC*I48M*Z1Ao(OlVkS7=a@e{2CcCf za8tu7Nji@8v18zmnr@1l48NcwZZ>>~cjGPxH)?6kM2$I%DXLy+z`g2Jd5;E1sZmN3 z23ZEsKo(fi;SkQX@R2!WDTcimVr@3x(EN;}VMFfu6;{)+vRT3{BUauNI;KTUSS7ucQ>ffXqe;2$Ja z#eT{AqUw`bxW&Y)XiEUE5-fpgpz;-tPwfr&aB*sP_!>8*s_+e-N=<@q@z1GK;64sb zD}evfSZhdo1|DDsmH(~Hf;m0&;CoH_UC&|g9Zt!N#LV>Uh<|Ey7>Eu7pGGuajeLzO z|HLKfN$@XhPOqiaFr?Qg_z_>~Re*8+M$C^2!>@YP!_O42h#vcX?f%Y*GWpe~ef2(iQy5U-rkLTmXF2uPqM`yi`;rrqEH)2ZA#U>CfKdD(gJ8ZOR$1(u<4 z;1{sG<-x$=04uQn;CNVx&kP<1tFW2m)p%?0c-nr|oOoD^BXSnQI=oDBEqBG3)i>&k^_`$GpZNW~34ce?eKYTfC#4#f#!wzg5kqMjd z{SmuhC%!Q9AZ){)qX=o4HYy)r4>smcggP490y1yy7oqp3okOW_DS2#;A1%#4j$cUS zGZPZ1d~w25DnFVKL*-8s#>soN;ie;5+1oPd$vuIvtL2x%V|u8^w`N|0hL#u0azt~3 zwp?j1qGJJ+dc5Y$khPQALdUkUt3@im%UirxeHR+A$vrJeYd{w^x|sORa+|_0a@3fI zqI(IK8!^3Rg6zA7nW@XTqsAH=N`gxF165R(^0C)WWbYkBP@3_2jg?NsuWLrae(XDE z863nja~h~Do?8n0@a?(z;fHw&*(lgHuHnK`k?waaPibjMo%e~q=LEMZZ)rY$oThHd zNsxJVU(yh+%bKo4bde*w$WdM7=q_?h7df_z9QRNj2#Wep0mafqwsw)@x!k2w3ChWq zKP>%JM+1Li*%0IR3~cm2vgMa$xq#2Em>QbdD9$_B9dsJ%RUc8Bkdpf`6(tKp+OKjb!F%oOUBxR8|1qed8(IzCQx@H=6KX zlOhIv%?O)_qu-RnT$PThLbt0bZ3?}u4w{phl|jF$_yC7OGI#P*i(M7f?!=xd*JPiT z5`6Jd*n8~(VKo(H6=m+SLTA|m2fh7MX?GWv;`FvK%sw2_86>3}+HPC0@o zLREv8zZkr~E*WcUEx2us5#mU*py!r>soW{fSnc()5U%34w~S!bq>pNi5YHuldPUa8 zAt6;u<+>rW9xjd9HjH42)tZ?3dJIddB5^V{y&kDg&MKZ+ScPMEq{>5+xtN-g@($;1 zGERD90i@!cH)>eg0u88JZ)H7ytBET&`eTm`5iEVa&*F#m8^T$yxewLojbW_!JfE}5 zGG4&;y64b;lgcs&;P}QEEZ7vu`gr>^c4sKw*<@sWJ(_=?%}K1^EIhP11YMh-V*M9t zXxx?_EX%2h+qZ+0fQD4wfZ3b{CZM*++8CjWd#60RnseF2GOtfJaneCrRR^N9 zF7Dg6S$t}F=Mw1$Pvg7$t>WbGIKtOm9Xx~g-YjL$&c}c4>4i%VJPuRv_JIcW+@a2A zp}xUXOg{8Dn>L-|rs0l5{h56mnYY7qoO3u63bE;MJ`~~o!?{q58AobZ$vDhBl7|~N zo5d8zv%XfnVFpfYd772hA)M-mpS3vI%>R&>fu6T+v$7vFanaFh`dQNoYuqjjIv0YW zC(>~Bu_&00&Bu;2=UVc~!Mfv1X_L>~SN0e9Rx;KN&2grc}?y@kg!r?WqFh z?rb-S)r|d>`_0xKY|gv5?^qhfo!-mlE~GXpPCsG52dAH6^PcoM=fHg2cIFgY&>1rh zN1rufXlpo*IqMI<#Y<;5uotHLyrZ$EH60e>xpU*-MO50VnCA%$YRkscZQaF1#M+Sd zL1=F8$rc0Vv=4_RxTt-(xD@(o6DDo9AND>UME5(}`DySn?l^yr?$*;UY@mzN4;P*h zU!zg_>B0b9aWRFiPe(8Ih2{8v$?WEf8ay;&*x`66? zR4=2t=M}PF=69*DFVm@n=~r^;E>(VIBHg9VUulN*SblY;v;kHFx~}G6=(P!Sy_tEf zDtZ%yRl7?D4JviG?G=u>_L+qhB~C|Ga>{0i!u0ErQ7K#C;r6eVIGhf*BYBRitRyvM zE2drVO^4!?>tpCfyZ3skSVvDi^?DROxE}7=0s3<7(`7vvOPpSDCrEsw-Ua#+Cl_{u z!S{W$0Rl?Os_8SR!(KeIreby@1eH~|9aR;DPCI=nr4ysHVh;ppH60yV;3m)&7P+e2 zdqJu$s+!#lX6v(uO5Vvmzz?SsHa5I~ju1GHAeIFxc%*9_$F1I-78{1^n%H*9@nfKG)8EtKgB*F|W*hl-O{T7gx?d1_lz! zU2`1AK~G;%osNnVpev~~UxO|y6o`jf8W?{9`;{H=HlxD|%aQBBtY9COPI=P{3t))#Me t7cXd@ad@0O^tb8wqB@sqx*K4|Z8N