From b8667afb9f61181508ffaa3766baf92702afb421 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Thu, 30 May 2024 21:33:01 +0200 Subject: [PATCH 01/11] Add first version of node / python deadlock fix. --- .../include/node_loader/node_loader_impl.h | 2 + .../node_loader/source/node_loader_impl.cpp | 59 +++++++++++++++++++ .../node_loader/source/node_loader_port.cpp | 21 ++----- 3 files changed, 67 insertions(+), 15 deletions(-) diff --git a/source/loaders/node_loader/include/node_loader/node_loader_impl.h b/source/loaders/node_loader/include/node_loader/node_loader_impl.h index 26290cecf..b85b7392c 100644 --- a/source/loaders/node_loader/include/node_loader/node_loader_impl.h +++ b/source/loaders/node_loader/include/node_loader/node_loader_impl.h @@ -55,6 +55,8 @@ NODE_LOADER_NO_EXPORT void node_loader_impl_exception(napi_env env, napi_status NODE_LOADER_NO_EXPORT void node_loader_impl_finalizer(napi_env env, napi_value v, void *data); +NODE_LOADER_NO_EXPORT void node_loader_impl_handle_promise(loader_impl_node node_impl, napi_env env, napi_deferred deferred, void *result, napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value), const char error_str[]); + NODE_LOADER_NO_EXPORT value node_loader_impl_napi_to_value(loader_impl_node node_impl, napi_env env, napi_value recv, napi_value v); NODE_LOADER_NO_EXPORT napi_value node_loader_impl_value_to_napi(loader_impl_node node_impl, napi_env env, value arg); diff --git a/source/loaders/node_loader/source/node_loader_impl.cpp b/source/loaders/node_loader/source/node_loader_impl.cpp index 245773d8a..05a8d2c28 100644 --- a/source/loaders/node_loader/source/node_loader_impl.cpp +++ b/source/loaders/node_loader/source/node_loader_impl.cpp @@ -513,6 +513,18 @@ struct loader_impl_async_future_delete_safe_type node_impl(node_impl), f(f), node_future(node_future) {} }; +struct loader_impl_async_handle_promise_safe_type +{ + loader_impl_node node_impl; + napi_deferred deferred; + void *result; + napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value); + const char *error_str; + + loader_impl_async_handle_promise_safe_type(loader_impl_node node_impl, napi_deferred deferred, void *result, napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value), const char error_str[]) : + node_impl(node_impl), deferred(deferred), result(result), deferred_fn(deferred_fn), error_str(error_str) {} +}; + struct loader_impl_async_destroy_safe_type { loader_impl_node node_impl; @@ -540,6 +552,7 @@ struct loader_impl_node_type loader_impl_threadsafe_type threadsafe_func_destroy; loader_impl_threadsafe_type threadsafe_future_await; loader_impl_threadsafe_type threadsafe_future_delete; + loader_impl_threadsafe_type threadsafe_handle_promise; loader_impl_threadsafe_type threadsafe_destroy; uv_thread_t thread; @@ -647,6 +660,8 @@ static value node_loader_impl_discover_function_safe(napi_env env, loader_impl_a static void node_loader_impl_discover_safe(napi_env env, loader_impl_async_discover_safe_type *discover_safe); +static void node_loader_impl_handle_promise_safe(napi_env env, loader_impl_async_handle_promise_safe_type *handle_promise_safe); + static void node_loader_impl_destroy_safe(napi_env env, loader_impl_async_destroy_safe_type *destroy_safe); static char *node_loader_impl_get_property_as_char(napi_env env, napi_value obj, const char *prop); @@ -2493,6 +2508,7 @@ void node_loader_impl_clear_safe(napi_env env, loader_impl_async_clear_safe_type napi_status status = napi_open_handle_scope(env, &handle_scope); node_loader_impl_exception(env, status); + /* Get function table object from reference */ status = napi_get_reference_value(env, clear_safe->node_impl->function_table_object_ref, &function_table_object); @@ -3271,6 +3287,30 @@ void node_loader_impl_discover_safe(napi_env env, loader_impl_async_discover_saf node_loader_impl_exception(env, status); } +void node_loader_impl_handle_promise_safe(napi_env env, loader_impl_async_handle_promise_safe_type *handle_promise_safe) +{ + napi_handle_scope handle_scope; + + /* Create scope */ + napi_status status = napi_open_handle_scope(env, &handle_scope); + + node_loader_impl_exception(env, status); + + /* Convert MetaCall value to napi value and call resolve or reject of NodeJS */ + napi_value js_result = node_loader_impl_value_to_napi(handle_promise_safe->node_impl, env, handle_promise_safe->result); + status = handle_promise_safe->deferred_fn(env, handle_promise_safe->deferred, js_result); + + if (status != napi_ok) + { + napi_throw_error(env, nullptr, handle_promise_safe->error_str); + } + + /* Close scope */ + status = napi_close_handle_scope(env, handle_scope); + + node_loader_impl_exception(env, status); +} + #if defined(_WIN32) && defined(_MSC_VER) && (_MSC_VER >= 1200) /* TODO: _Ret_maybenull_ HMODULE WINAPI GetModuleHandleW(_In_opt_ LPCWSTR lpModuleName); */ _Ret_maybenull_ HMODULE WINAPI get_module_handle_a_hook(_In_opt_ LPCSTR lpModuleName) @@ -3361,6 +3401,7 @@ void *node_loader_impl_register(void *node_impl_ptr, void *env_ptr, void *functi node_impl->threadsafe_func_destroy.initialize(env, "node_loader_impl_async_func_destroy_safe", &node_loader_impl_func_destroy_safe); node_impl->threadsafe_future_await.initialize(env, "node_loader_impl_async_future_await_safe", &node_loader_impl_future_await_safe); node_impl->threadsafe_future_delete.initialize(env, "node_loader_impl_async_future_delete_safe", &node_loader_impl_future_delete_safe); + node_impl->threadsafe_handle_promise.initialize(env, "node_loader_impl_async_handle_promise_safe", &node_loader_impl_handle_promise_safe); node_impl->threadsafe_destroy.initialize(env, "node_loader_impl_async_destroy_safe", &node_loader_impl_destroy_safe); } @@ -4030,6 +4071,23 @@ int node_loader_impl_discover(loader_impl impl, loader_handle handle, context ct return discover_safe.result; } +void node_loader_impl_handle_promise(loader_impl_node node_impl, napi_env env, napi_deferred deferred, void *result, napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value), const char error_str[]) +{ + loader_impl_async_handle_promise_safe_type handle_promise_safe(node_impl, deferred, result, deferred_fn, error_str); + + /* Check if we are in the JavaScript thread */ + if (node_impl->js_thread_id == std::this_thread::get_id()) + { + /* We are already in the V8 thread, we can call safely */ + node_loader_impl_handle_promise_safe(env, &handle_promise_safe); + } + else + { + /* Submit the task to the async queue */ + loader_impl_threadsafe_invoke_type invoke(node_impl->threadsafe_handle_promise, handle_promise_safe); + } +} + #define container_of(ptr, type, member) \ (type *)((char *)(ptr) - (char *)&((type *)0)->member) @@ -4270,6 +4328,7 @@ void node_loader_impl_destroy_safe_impl(loader_impl_node node_impl, napi_env env node_impl->threadsafe_func_destroy.abort(env); node_impl->threadsafe_future_await.abort(env); node_impl->threadsafe_future_delete.abort(env); + node_impl->threadsafe_handle_promise.abort(env); } /* Clear persistent references */ diff --git a/source/loaders/node_loader/source/node_loader_port.cpp b/source/loaders/node_loader/source/node_loader_port.cpp index 6806c240c..ad8952230 100644 --- a/source/loaders/node_loader/source/node_loader_port.cpp +++ b/source/loaders/node_loader/source/node_loader_port.cpp @@ -198,14 +198,11 @@ napi_value node_loader_port_metacall_await(napi_env env, napi_callback_info info ctx->env = env; auto resolve = [](void *result, void *data) -> void * { + static const char promise_error_str[] = "Failed to resolve the promise"; + promise_context_type *ctx = static_cast(data); - napi_value js_result = node_loader_impl_value_to_napi(ctx->node_impl, ctx->env, result); - napi_status status = napi_resolve_deferred(ctx->env, ctx->deferred, js_result); - if (status != napi_ok) - { - napi_throw_error(ctx->env, nullptr, "Failed to resolve the promise"); - } + node_loader_impl_handle_promise(ctx->node_impl, ctx->env, ctx->deferred, result, &napi_resolve_deferred, promise_error_str); delete ctx; @@ -213,14 +210,11 @@ napi_value node_loader_port_metacall_await(napi_env env, napi_callback_info info }; auto reject = [](void *result, void *data) -> void * { + static const char promise_error_str[] = "Failed to reject the promise"; + promise_context_type *ctx = static_cast(data); - napi_value js_result = node_loader_impl_value_to_napi(ctx->node_impl, ctx->env, result); - napi_status status = napi_reject_deferred(ctx->env, ctx->deferred, js_result); - if (status != napi_ok) - { - napi_throw_error(ctx->env, nullptr, "Failed to reject the promise"); - } + node_loader_impl_handle_promise(ctx->node_impl, ctx->env, ctx->deferred, result, &napi_reject_deferred, promise_error_str); delete ctx; @@ -230,15 +224,12 @@ napi_value node_loader_port_metacall_await(napi_env env, napi_callback_info info /* Await to the function */ void *ret = metacall_await_s(name, args, argc - 1, resolve, reject, ctx); - /* TODO: Is this needed? */ - /* if (metacall_value_id(ret) == METACALL_THROWABLE) { napi_value result = node_loader_impl_value_to_napi(node_impl, env, ret); napi_throw(env, result); } - */ /* Release current reference of the environment */ // node_loader_impl_env(node_impl, nullptr); From 92adb18ffd76f3250e8366104a8e50950dc4a89e Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Fri, 7 Jun 2024 17:11:39 +0200 Subject: [PATCH 02/11] Initial version of node/py await. --- .../include/node_loader/node_loader_impl.h | 2 +- .../node_loader/source/node_loader_impl.cpp | 242 +++++++++++++----- .../node_loader/source/node_loader_port.cpp | 70 +---- 3 files changed, 184 insertions(+), 130 deletions(-) diff --git a/source/loaders/node_loader/include/node_loader/node_loader_impl.h b/source/loaders/node_loader/include/node_loader/node_loader_impl.h index b85b7392c..6aa1f3448 100644 --- a/source/loaders/node_loader/include/node_loader/node_loader_impl.h +++ b/source/loaders/node_loader/include/node_loader/node_loader_impl.h @@ -55,7 +55,7 @@ NODE_LOADER_NO_EXPORT void node_loader_impl_exception(napi_env env, napi_status NODE_LOADER_NO_EXPORT void node_loader_impl_finalizer(napi_env env, napi_value v, void *data); -NODE_LOADER_NO_EXPORT void node_loader_impl_handle_promise(loader_impl_node node_impl, napi_env env, napi_deferred deferred, void *result, napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value), const char error_str[]); +NODE_LOADER_NO_EXPORT napi_value node_loader_impl_promise_await(loader_impl_node node_impl, napi_env env, const char *name, value *args, size_t size); NODE_LOADER_NO_EXPORT value node_loader_impl_napi_to_value(loader_impl_node node_impl, napi_env env, napi_value recv, napi_value v); diff --git a/source/loaders/node_loader/source/node_loader_impl.cpp b/source/loaders/node_loader/source/node_loader_impl.cpp index 05a8d2c28..5d55f1959 100644 --- a/source/loaders/node_loader/source/node_loader_impl.cpp +++ b/source/loaders/node_loader/source/node_loader_impl.cpp @@ -284,12 +284,37 @@ void node_loader_impl_func_call_js_safe(napi_env env, napi_value js_callback, vo // async_safe->args->node_impl->env = nullptr; } +template +void node_loader_impl_func_call_js_async_safe(napi_env env, napi_value js_callback, void *context, void *data) +{ + (void)js_callback; + + if (env == nullptr || context == nullptr || data == nullptr) + { + log_write("metacall", LOG_LEVEL_ERROR, "Invalid arguments passed to js thread async safe function"); + return; + } + + T *args = static_cast(data); + node_loader_impl_func_call_js_safe_cast safe_cast(context); + + /* Store environment for reentrant calls */ + args->node_impl->env = env; + + /* Call to the implementation function */ + safe_cast.func_ptr(env, args); + + /* Clear environment */ + // async_safe->args->node_impl->env = nullptr; +} + static napi_value node_loader_impl_async_threadsafe_empty(napi_env, napi_callback_info) { /* This is a dirty hack in order to make the threadsafe API work properly, * as soon as possible it will be good to return to the old method we used in NodeJS 8, * it was better than this API */ + return nullptr; } @@ -513,18 +538,6 @@ struct loader_impl_async_future_delete_safe_type node_impl(node_impl), f(f), node_future(node_future) {} }; -struct loader_impl_async_handle_promise_safe_type -{ - loader_impl_node node_impl; - napi_deferred deferred; - void *result; - napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value); - const char *error_str; - - loader_impl_async_handle_promise_safe_type(loader_impl_node node_impl, napi_deferred deferred, void *result, napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value), const char error_str[]) : - node_impl(node_impl), deferred(deferred), result(result), deferred_fn(deferred_fn), error_str(error_str) {} -}; - struct loader_impl_async_destroy_safe_type { loader_impl_node node_impl; @@ -552,7 +565,6 @@ struct loader_impl_node_type loader_impl_threadsafe_type threadsafe_func_destroy; loader_impl_threadsafe_type threadsafe_future_await; loader_impl_threadsafe_type threadsafe_future_delete; - loader_impl_threadsafe_type threadsafe_handle_promise; loader_impl_threadsafe_type threadsafe_destroy; uv_thread_t thread; @@ -584,6 +596,73 @@ struct loader_impl_node_type loader_impl impl; }; +template +struct loader_impl_threadsafe_async_type +{ + uv_async_t async_handle; + bool initialized; + + int initialize(loader_impl_node node_impl, void (*async_cb)(uv_async_t *)) + { + int result = uv_async_init(node_impl->thread_loop, &async_handle, async_cb); + + initialized = (result == 0); + + return result; + } + + void invoke(T *data) + { + if (initialized) + { + async_handle.data = static_cast(data); + uv_async_send(&async_handle); + } + } + + void close(void (*close_cb)(uv_handle_t *handle)) + { + if (initialized) + { + union + { + uv_handle_t *handle; + uv_async_t *async; + } handle_cast; + + handle_cast.async = &async_handle; + + uv_close(handle_cast.handle, close_cb); + } + } +}; + +struct loader_impl_async_handle_promise_safe_type +{ + loader_impl_node node_impl; + napi_env env; + napi_deferred deferred; + void *result; + napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value); + const char *error_str; + loader_impl_threadsafe_async_type threadsafe_async; + + loader_impl_async_handle_promise_safe_type(loader_impl_node node_impl, napi_env env) : + node_impl(node_impl), env(env), result(NULL) {} + + ~loader_impl_async_handle_promise_safe_type() + { + threadsafe_async.close([](uv_handle_t *handle) { + loader_impl_async_handle_promise_safe_type *handle_promise_safe = static_cast(handle->data); + + if (handle_promise_safe->result != NULL) + { + metacall_value_destroy(handle_promise_safe->result); + } + }); + } +}; + typedef napi_value (*function_resolve_trampoline)(loader_impl_node, napi_env, function_resolve_callback, napi_value, napi_value, void *); typedef napi_value (*function_reject_trampoline)(loader_impl_node, napi_env, function_reject_callback, napi_value, napi_value, void *); @@ -3305,6 +3384,9 @@ void node_loader_impl_handle_promise_safe(napi_env env, loader_impl_async_handle napi_throw_error(env, nullptr, handle_promise_safe->error_str); } + /* Close the handle */ + delete handle_promise_safe; + /* Close scope */ status = napi_close_handle_scope(env, handle_scope); @@ -3401,7 +3483,6 @@ void *node_loader_impl_register(void *node_impl_ptr, void *env_ptr, void *functi node_impl->threadsafe_func_destroy.initialize(env, "node_loader_impl_async_func_destroy_safe", &node_loader_impl_func_destroy_safe); node_impl->threadsafe_future_await.initialize(env, "node_loader_impl_async_future_await_safe", &node_loader_impl_future_await_safe); node_impl->threadsafe_future_delete.initialize(env, "node_loader_impl_async_future_delete_safe", &node_loader_impl_future_delete_safe); - node_impl->threadsafe_handle_promise.initialize(env, "node_loader_impl_async_handle_promise_safe", &node_loader_impl_handle_promise_safe); node_impl->threadsafe_destroy.initialize(env, "node_loader_impl_async_destroy_safe", &node_loader_impl_destroy_safe); } @@ -4071,23 +4152,95 @@ int node_loader_impl_discover(loader_impl impl, loader_handle handle, context ct return discover_safe.result; } -void node_loader_impl_handle_promise(loader_impl_node node_impl, napi_env env, napi_deferred deferred, void *result, napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value), const char error_str[]) +void node_loader_impl_handle_promise(loader_impl_async_handle_promise_safe_type *handle_promise_safe, void *result, napi_status (*deferred_fn)(napi_env, napi_deferred, napi_value), const char error_str[]) { - loader_impl_async_handle_promise_safe_type handle_promise_safe(node_impl, deferred, result, deferred_fn, error_str); + handle_promise_safe->result = metacall_value_copy(result); + handle_promise_safe->deferred_fn = deferred_fn; + handle_promise_safe->error_str = error_str; /* Check if we are in the JavaScript thread */ - if (node_impl->js_thread_id == std::this_thread::get_id()) + if (handle_promise_safe->node_impl->js_thread_id == std::this_thread::get_id()) { /* We are already in the V8 thread, we can call safely */ - node_loader_impl_handle_promise_safe(env, &handle_promise_safe); + node_loader_impl_handle_promise_safe(handle_promise_safe->env, handle_promise_safe); } else { /* Submit the task to the async queue */ - loader_impl_threadsafe_invoke_type invoke(node_impl->threadsafe_handle_promise, handle_promise_safe); + if (handle_promise_safe->threadsafe_async.initialize(handle_promise_safe->node_impl, [](uv_async_t *handle) { + loader_impl_async_handle_promise_safe_type *handle_promise_safe = static_cast(handle->data); + node_loader_impl_handle_promise_safe(handle_promise_safe->env, handle_promise_safe); + }) != 0) + { + log_write("metacall", LOG_LEVEL_ERROR, "Filed to initialize promise safe async handle"); + } + else + { + handle_promise_safe->threadsafe_async.invoke(handle_promise_safe); + } } } +napi_value node_loader_impl_promise_await(loader_impl_node node_impl, napi_env env, const char *name, value *args, size_t size) +{ + loader_impl_async_handle_promise_safe_type *handle_promise_safe = new loader_impl_async_handle_promise_safe_type(node_impl, env); + + if (handle_promise_safe == nullptr) + { + napi_throw_error(env, nullptr, "Failed to allocate the promise context"); + + return nullptr; + } + + napi_value promise; + + /* Create the promise */ + napi_status status = napi_create_promise(env, &handle_promise_safe->deferred, &promise); + + if (status != napi_ok) + { + napi_throw_error(env, nullptr, "Failed to create the promise"); + + delete handle_promise_safe; + + return nullptr; + } + + auto resolve = [](void *result, void *data) -> void * { + static const char promise_error_str[] = "Failed to resolve the promise"; + + loader_impl_async_handle_promise_safe_type *handle_promise_safe = static_cast(data); + + node_loader_impl_handle_promise(handle_promise_safe, result, &napi_resolve_deferred, promise_error_str); + + return NULL; + }; + + auto reject = [](void *result, void *data) -> void * { + static const char promise_error_str[] = "Failed to reject the promise"; + + loader_impl_async_handle_promise_safe_type *handle_promise_safe = static_cast(data); + + node_loader_impl_handle_promise(handle_promise_safe, result, &napi_reject_deferred, promise_error_str); + + return NULL; + }; + + /* Await to the function */ + void *ret = metacall_await_s(name, args, size, resolve, reject, handle_promise_safe); + + if (metacall_value_id(ret) == METACALL_THROWABLE) + { + napi_value result = node_loader_impl_value_to_napi(node_impl, env, ret); + + napi_throw(env, result); + } + + node_loader_impl_finalizer(env, promise, ret); + + return promise; +} + #define container_of(ptr, type, member) \ (type *)((char *)(ptr) - (char *)&((type *)0)->member) @@ -4311,10 +4464,16 @@ void node_loader_impl_destroy_safe_impl(loader_impl_node node_impl, napi_env env { uint32_t ref_count = 0; napi_status status; + napi_handle_scope handle_scope; /* Destroy children loaders */ loader_unload_children(node_impl->impl); + /* Create scope */ + status = napi_open_handle_scope(env, &handle_scope); + + node_loader_impl_exception(env, status); + /* Clear thread safe functions except by destroy one */ { node_impl->threadsafe_initialize.abort(env); @@ -4328,7 +4487,6 @@ void node_loader_impl_destroy_safe_impl(loader_impl_node node_impl, napi_env env node_impl->threadsafe_func_destroy.abort(env); node_impl->threadsafe_future_await.abort(env); node_impl->threadsafe_future_delete.abort(env); - node_impl->threadsafe_handle_promise.abort(env); } /* Clear persistent references */ @@ -4358,48 +4516,10 @@ void node_loader_impl_destroy_safe_impl(loader_impl_node node_impl, napi_env env node_loader_impl_exception(env, status); - /* Clear event loop */ - { - /* Stop event loop */ - uv_stop(node_impl->thread_loop); - - /* Clear event loop */ - /* uv_walk(node_impl->thread_loop, node_loader_impl_walk, NULL); */ - -#if 0 - /* TODO: For some reason, this deadlocks in NodeJS benchmark when mixing sync and async calls. - * It should be reviewed carefully and detect if NodeJS is finalizing properly on multiple cases. - * Disable it for now in order to make tests pass. - */ - while (uv_run(node_impl->thread_loop, UV_RUN_DEFAULT) != 0) - #if (!defined(NDEBUG) || defined(DEBUG) || defined(_DEBUG) || defined(__DEBUG) || defined(__DEBUG__)) - { - node_loader_impl_print_handles(node_impl); - } - #else - ; - #endif - - /* Destroy node loop */ - if (uv_loop_alive(node_impl->thread_loop) != 0) - { - /* TODO: Make logs thread safe */ - /* log_write("metacall", LOG_LEVEL_ERROR, "NodeJS event loop should not be alive"); */ - printf("NodeJS Loader Error: NodeJS event loop should not be alive\n"); - fflush(stdout); - } -#endif + /* Close scope */ + status = napi_close_handle_scope(env, handle_scope); - /* This evaluates to true always due to stdin and stdout handles, - which are closed anyway on thread join. So it is removed by now. */ - if (uv_loop_close(node_impl->thread_loop) != UV_EBUSY) - { - /* TODO: Make logs thread safe */ - /* log_write("metacall", LOG_LEVEL_ERROR, "NodeJS event loop should not be busy"); */ - printf("NodeJS Loader Error: NodeJS event loop should be busy\n"); - fflush(stdout); - } - } + node_loader_impl_exception(env, status); /* NodeJS Loader needs to register that it is destroyed, because after this step * some destructors can be still triggered, before the node_loader->destroy() has diff --git a/source/loaders/node_loader/source/node_loader_port.cpp b/source/loaders/node_loader/source/node_loader_port.cpp index ad8952230..c77592ff6 100644 --- a/source/loaders/node_loader/source/node_loader_port.cpp +++ b/source/loaders/node_loader/source/node_loader_port.cpp @@ -36,13 +36,6 @@ #include -struct promise_context_type -{ - loader_impl_node node_impl; - napi_env env; - napi_deferred deferred; -}; - static const loader_tag node_loader_tag = "node"; napi_value node_loader_port_metacall(napi_env env, napi_callback_info info) @@ -171,65 +164,8 @@ napi_value node_loader_port_metacall_await(napi_env env, napi_callback_info info args[args_count - 1] = node_loader_impl_napi_to_value(node_impl, env, recv, argv[args_count]); } - promise_context_type *ctx = new promise_context_type(); - - if (ctx == nullptr) - { - napi_throw_error(env, nullptr, "Failed to allocate the promise context"); - - return nullptr; - } - - napi_value promise; - - /* Create the promise */ - status = napi_create_promise(env, &ctx->deferred, &promise); - - if (status != napi_ok) - { - napi_throw_error(env, nullptr, "Failed to create the promise"); - - delete ctx; - - return nullptr; - } - - ctx->node_impl = node_impl; - ctx->env = env; - - auto resolve = [](void *result, void *data) -> void * { - static const char promise_error_str[] = "Failed to resolve the promise"; - - promise_context_type *ctx = static_cast(data); - - node_loader_impl_handle_promise(ctx->node_impl, ctx->env, ctx->deferred, result, &napi_resolve_deferred, promise_error_str); - - delete ctx; - - return NULL; - }; - - auto reject = [](void *result, void *data) -> void * { - static const char promise_error_str[] = "Failed to reject the promise"; - - promise_context_type *ctx = static_cast(data); - - node_loader_impl_handle_promise(ctx->node_impl, ctx->env, ctx->deferred, result, &napi_reject_deferred, promise_error_str); - - delete ctx; - - return NULL; - }; - - /* Await to the function */ - void *ret = metacall_await_s(name, args, argc - 1, resolve, reject, ctx); - - if (metacall_value_id(ret) == METACALL_THROWABLE) - { - napi_value result = node_loader_impl_value_to_napi(node_impl, env, ret); - - napi_throw(env, result); - } + /* Call to metacall await and wrap the promise into NodeJS land */ + napi_value promise = node_loader_impl_promise_await(node_impl, env, name, args, argc - 1); /* Release current reference of the environment */ // node_loader_impl_env(node_impl, nullptr); @@ -239,8 +175,6 @@ napi_value node_loader_port_metacall_await(napi_env env, napi_callback_info info metacall_value_destroy(args[args_count]); } - node_loader_impl_finalizer(env, promise, ret); - delete[] argv; delete[] args; delete[] name; From c3b1385be4aa0fe34b553e7c8dbf97a555a80a02 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Tue, 11 Jun 2024 19:12:11 +0200 Subject: [PATCH 03/11] Add more complex test for mixing node and python await. --- source/tests/CMakeLists.txt | 5 +- .../CMakeLists.txt | 152 ++++++++++++++++++ .../source/main.cpp | 28 ++++ ...tacall_node_python_await_extended_test.cpp | 91 +++++++++++ 4 files changed, 274 insertions(+), 2 deletions(-) create mode 100644 source/tests/metacall_node_python_await_extended_test/CMakeLists.txt create mode 100644 source/tests/metacall_node_python_await_extended_test/source/main.cpp create mode 100644 source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp diff --git a/source/tests/CMakeLists.txt b/source/tests/CMakeLists.txt index 9a1202a56..b6c702669 100644 --- a/source/tests/CMakeLists.txt +++ b/source/tests/CMakeLists.txt @@ -150,6 +150,7 @@ add_subdirectory(metacall_node_fail_load_leak_test) add_subdirectory(metacall_node_typescript_test) add_subdirectory(metacall_node_python_async_after_destroy_test) add_subdirectory(metacall_node_python_await_test) +add_subdirectory(metacall_node_python_await_extended_test) add_subdirectory(metacall_node_python_exception_test) add_subdirectory(metacall_node_clear_mem_test) add_subdirectory(metacall_node_async_resources_test) @@ -190,9 +191,9 @@ add_subdirectory(metacall_python_relative_path_test) add_subdirectory(metacall_python_without_functions_test) add_subdirectory(metacall_python_builtins_test) add_subdirectory(metacall_python_async_test) -# TODO: add_subdirectory(metacall_python_await_test) +# TODO: add_subdirectory(metacall_python_await_test) # TODO: Implement metacall_await in Python Port add_subdirectory(metacall_python_exception_test) -# TODO: add_subdirectory(metacall_python_node_await_test) +# TODO: add_subdirectory(metacall_python_node_await_test) # TODO: Implement metacall_await in Python Port add_subdirectory(metacall_python_without_env_vars_test) add_subdirectory(metacall_map_test) add_subdirectory(metacall_map_await_test) diff --git a/source/tests/metacall_node_python_await_extended_test/CMakeLists.txt b/source/tests/metacall_node_python_await_extended_test/CMakeLists.txt new file mode 100644 index 000000000..af1945174 --- /dev/null +++ b/source/tests/metacall_node_python_await_extended_test/CMakeLists.txt @@ -0,0 +1,152 @@ +# Check if this loader is enabled +if(NOT OPTION_BUILD_LOADERS OR NOT OPTION_BUILD_LOADERS_NODE OR NOT OPTION_BUILD_LOADERS_PY OR NOT OPTION_BUILD_PORTS OR NOT OPTION_BUILD_PORTS_NODE) + return() +endif() + +# +# Executable name and options +# + +# Target name +set(target metacall-node-python-await-extended-test) +message(STATUS "Test ${target}") + +# +# Compiler warnings +# + +include(Warnings) + +# +# Compiler security +# + +include(SecurityFlags) + +# +# Sources +# + +set(include_path "${CMAKE_CURRENT_SOURCE_DIR}/include/${target}") +set(source_path "${CMAKE_CURRENT_SOURCE_DIR}/source") + +set(sources + ${source_path}/main.cpp + ${source_path}/metacall_node_python_await_extended_test.cpp +) + +# Group source files +set(header_group "Header Files (API)") +set(source_group "Source Files") +source_group_by_path(${include_path} "\\\\.h$|\\\\.hpp$" + ${header_group} ${headers}) +source_group_by_path(${source_path} "\\\\.cpp$|\\\\.c$|\\\\.h$|\\\\.hpp$" + ${source_group} ${sources}) + +# +# Create executable +# + +# Build executable +add_executable(${target} + ${sources} +) + +# Create namespaced alias +add_executable(${META_PROJECT_NAME}::${target} ALIAS ${target}) + +# +# Project options +# + +set_target_properties(${target} + PROPERTIES + ${DEFAULT_PROJECT_OPTIONS} + FOLDER "${IDE_FOLDER}" +) + +# +# Include directories +# + +target_include_directories(${target} + PRIVATE + ${DEFAULT_INCLUDE_DIRECTORIES} + ${PROJECT_BINARY_DIR}/source/include +) + +# +# Libraries +# + +target_link_libraries(${target} + PRIVATE + ${DEFAULT_LIBRARIES} + + GTest + + ${META_PROJECT_NAME}::metacall +) + +# +# Compile definitions +# + +target_compile_definitions(${target} + PRIVATE + ${DEFAULT_COMPILE_DEFINITIONS} + + # NodeJS Port path + METACALL_NODE_PORT_PATH="${CMAKE_SOURCE_DIR}/source/ports/node_port/index.js" +) + +# +# Compile options +# + +target_compile_options(${target} + PRIVATE + ${DEFAULT_COMPILE_OPTIONS} +) + +# +# Linker options +# + +target_link_libraries(${target} + PRIVATE + ${DEFAULT_LINKER_OPTIONS} +) + +# +# Define test +# + +add_test(NAME ${target} + COMMAND $ +) + +# +# Define dependencies +# + +add_dependencies(${target} + node_loader + py_loader +) + +# +# Define test properties +# + +set_property(TEST ${target} + PROPERTY LABELS ${target} +) + +include(TestEnvironmentVariables) + +test_environment_variables(${target} + "" + ${TESTS_ENVIRONMENT_VARIABLES} + ${PY_DEBUG_ENVIRONMENT_VARIABLES} +) diff --git a/source/tests/metacall_node_python_await_extended_test/source/main.cpp b/source/tests/metacall_node_python_await_extended_test/source/main.cpp new file mode 100644 index 000000000..11ddf3f59 --- /dev/null +++ b/source/tests/metacall_node_python_await_extended_test/source/main.cpp @@ -0,0 +1,28 @@ +/* + * MetaCall Library by Parra Studios + * A library for providing a foreign function interface calls. + * + * Copyright (C) 2016 - 2024 Vicente Eduardo Ferrer Garcia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +int main(int argc, char *argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + + return RUN_ALL_TESTS(); +} diff --git a/source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp b/source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp new file mode 100644 index 000000000..3ce6d719d --- /dev/null +++ b/source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp @@ -0,0 +1,91 @@ +/* + * MetaCall Library by Parra Studios + * A library for providing a foreign function interface calls. + * + * Copyright (C) 2016 - 2024 Vicente Eduardo Ferrer Garcia + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include +#include + +class metacall_node_python_await_extended_test : public testing::Test +{ +public: +}; + +TEST_F(metacall_node_python_await_extended_test, DefaultConstructor) +{ + metacall_print_info(); + + ASSERT_EQ((int)0, (int)metacall_initialize()); + +/* NodeJS & Python */ +#if defined(OPTION_BUILD_LOADERS_NODE) && defined(OPTION_BUILD_LOADERS_PY) + { + static const char buffer[] = + /* NodeJS */ + "const { metacall_await, metacall_load_from_memory, metacall_inspect } = require('" METACALL_NODE_PORT_PATH "');\n" + "const util = require('util')\n" + "metacall_load_from_memory('py', `" + /* Python */ + "import sys\n" + "import threading\n" + "async def python_simple(n):\n" + "\tprint('inside python_simple', threading.current_thread().ident, ':', n)\n" + "\tsys.stdout.flush()\n" + "\treturn n\n" + // "import asyncio\n" + // "async def python_simple(n):\n" + // " await asyncio.sleep(1)\n" + // " return n\n" + "`);\n" + /* Debug */ + "console.log('--------------------------------------------------------------------')\n" + "console.log(util.inspect(metacall_inspect(), false, null, true))\n" + "console.log('--------------------------------------------------------------------')\n" + /* NodeJS Check */ + "const size = 10000;\n" + "let buffer = new SharedArrayBuffer(4);\n" + "let int32 = new Int32Array(buffer);\n" + "Atomics.store(int32, 0, 0);\n" + "process.on('exit', () => {\n" + " if (Atomics.load(int32, 0) != size) {\n" + " process.exit(3);\n" + " }\n" + "});\n" + /* NodeJS Promise */ + "for (let i = 0; i < size; i++) {\n" + " metacall_await('python_simple', 32).then(v => {\n" + " console.log('RESULT:', v, Atomics.load(int32, 0));\n" + " if (v !== 32) {\n" + " process.exit(1);\n" + " }\n" + " Atomics.add(int32, 0, 1);\n" + " }).catch(v => {\n" + " console.log('ERROR:', v);\n" + " process.exit(2);\n" + " });\n" + "}\n"; + + ASSERT_EQ((int)0, (int)metacall_load_from_memory("node", buffer, sizeof(buffer), NULL)); + } +#endif /* OPTION_BUILD_LOADERS_NODE && OPTION_BUILD_LOADERS_PY */ + + EXPECT_EQ((int)0, (int)metacall_destroy()); +} From c25e6edd18f10f164e3ba54a725c009db3e7eac1 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 18:12:01 +0200 Subject: [PATCH 04/11] Trying to solve more issues from nodejs. --- .github/workflows/docker-hub.yml | 4 +- .../node_loader/bootstrap/lib/bootstrap.js | 15 ++- .../node_loader/source/node_loader_impl.cpp | 103 +++++++++++------- ...tacall_node_python_await_extended_test.cpp | 5 +- 4 files changed, 76 insertions(+), 51 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index 12375d985..db4012ce6 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -8,6 +8,7 @@ on: push: branches: - master + - develop tags: - 'v*.*.*' @@ -44,9 +45,6 @@ jobs: bash ./docker-compose.sh push elif [[ "${{ contains(github.ref, 'refs/tags/') }}" = true ]]; then bash ./docker-compose.sh version - else - echo "Failed to push the docker images" - exit 1 fi - name: Logout from DockerHub diff --git a/source/loaders/node_loader/bootstrap/lib/bootstrap.js b/source/loaders/node_loader/bootstrap/lib/bootstrap.js index 650a2b7ab..5582ab48e 100644 --- a/source/loaders/node_loader/bootstrap/lib/bootstrap.js +++ b/source/loaders/node_loader/bootstrap/lib/bootstrap.js @@ -6,26 +6,29 @@ const path = require('path'); const util = require('util'); const fs = require('fs'); -/* Require the JavaScript parser */ +// Require the JavaScript parser const espree = require(path.join(__dirname, 'node_modules', 'espree')); const node_require = Module.prototype.require; const node_resolve = require.resolve; const node_cache = require.cache; -/* Store in the module prototype the original functions for future use in derived loaders like TypeScript */ +// Store in the module prototype the original functions for future use in derived loaders like TypeScript Module.prototype.node_require = node_require; Module.prototype.node_resolve = node_resolve; Module.prototype.node_cache = node_cache; function node_loader_trampoline_initialize(loader_library_path) { + // Restore the argv (this is used for tricking node::Start method) + process.argv = [ process.argv[0] ]; + // Add current execution directory to the execution paths node_loader_trampoline_execution_path(process.cwd()); const paths = [ - /* Local version of MetaCall NodeJS Port */ + // Local version of MetaCall NodeJS Port 'metacall', - /* Optionally, use loader library path for global installed NodeJS Port */ + // Optionally, use loader library path for global installed NodeJS Port ...loader_library_path ? [ path.join(loader_library_path, 'node_modules', 'metacall', 'index.js') ] : [], ]; @@ -333,7 +336,7 @@ function node_loader_trampoline_discover(handle) { } function node_loader_trampoline_test(obj) { - /* Imporant: never trigger an async resource in this function */ + // Imporant: never trigger an async resource in this function if (obj !== undefined) { fs.writeSync(process.stdout.fd, `${util.inspect(obj, false, null, true)}\n`); } @@ -411,7 +414,7 @@ module.exports = ((impl, ptr) => { throw new Error('Process arguments (process.argv[2], process.argv[3]) not defined.'); } - /* Get trampoline from list of linked bindings */ + // Get trampoline from list of linked bindings const trampoline = process._linkedBinding('node_loader_trampoline_module'); const node_loader_ptr = trampoline.register(impl, ptr, { diff --git a/source/loaders/node_loader/source/node_loader_impl.cpp b/source/loaders/node_loader/source/node_loader_impl.cpp index 5d55f1959..eb0150e18 100644 --- a/source/loaders/node_loader/source/node_loader_impl.cpp +++ b/source/loaders/node_loader/source/node_loader_impl.cpp @@ -99,9 +99,15 @@ extern char **environ; #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif -#include +/* NodeJS Includes */ #include +#ifdef NAPI_VERSION + #undef NAPI_VERSION +#endif + +#include + #include /* version: 6.2.414.50 */ #ifdef ENABLE_DEBUGGER_SUPPORT @@ -650,7 +656,7 @@ struct loader_impl_async_handle_promise_safe_type loader_impl_async_handle_promise_safe_type(loader_impl_node node_impl, napi_env env) : node_impl(node_impl), env(env), result(NULL) {} - ~loader_impl_async_handle_promise_safe_type() + void destroy() { threadsafe_async.close([](uv_handle_t *handle) { loader_impl_async_handle_promise_safe_type *handle_promise_safe = static_cast(handle->data); @@ -659,6 +665,8 @@ struct loader_impl_async_handle_promise_safe_type { metacall_value_destroy(handle_promise_safe->result); } + + delete handle_promise_safe; }); } }; @@ -772,6 +780,40 @@ static HMODULE (*get_module_handle_a_ptr)(_In_opt_ LPCSTR) = NULL; /* TODO: Impl /* -- Methods -- */ +#if 1 // NODE_MAJOR_VERSION < 18 + #if NODE_MAJOR_VERSION >= 12 + #define node_loader_impl_register_module_id node::ModuleFlags::kLinked + #else + #define node_loader_impl_register_module_id 0x02 /* NM_F_LINKED */ + #endif + + #define node_loader_impl_register_module(name, fn) \ + do \ + { \ + static napi_module node_loader_module = { \ + NAPI_MODULE_VERSION, \ + node_loader_impl_register_module_id, \ + __FILE__, \ + fn, \ + name, \ + NULL, \ + { 0 } \ + }; \ + napi_module_register(&node_loader_module); \ + } while (0) + +void node_loader_impl_register_linked_bindings() +{ + /* Initialize Node Loader Trampoline */ + node_loader_impl_register_module("node_loader_trampoline_module", node_loader_trampoline_initialize); + + /* Initialize Node Loader Port */ + node_loader_impl_register_module("node_loader_port_module", node_loader_port_initialize); +} +#else +// TODO: New register implementation +#endif + void node_loader_impl_exception(napi_env env, napi_status status) { if (status != napi_ok) @@ -3385,7 +3427,7 @@ void node_loader_impl_handle_promise_safe(napi_env env, loader_impl_async_handle } /* Close the handle */ - delete handle_promise_safe; + handle_promise_safe->destroy(); /* Close scope */ status = napi_close_handle_scope(env, handle_scope); @@ -3445,6 +3487,7 @@ void *node_loader_impl_register(void *node_impl_ptr, void *env_ptr, void *functi napi_value function_table_object; napi_value global; napi_status status; + napi_handle_scope handle_scope; /* Lock node implementation mutex */ uv_mutex_lock(&node_impl->mutex); @@ -3456,6 +3499,11 @@ void *node_loader_impl_register(void *node_impl_ptr, void *env_ptr, void *functi env = static_cast(env_ptr); function_table_object = static_cast(function_table_object_ptr); + /* Create scope */ + status = napi_open_handle_scope(env, &handle_scope); + + node_loader_impl_exception(env, status); + /* Make global object persistent */ status = napi_get_global(env, &global); @@ -3548,6 +3596,11 @@ void *node_loader_impl_register(void *node_impl_ptr, void *env_ptr, void *functi get_module_handle_a_ptr = (HMODULE(*)(_In_opt_ LPCSTR))node_loader_hook_import_address_table("kernel32.dll", "GetModuleHandleA", &get_module_handle_a_hook); #endif + /* Close scope */ + status = napi_close_handle_scope(env, handle_scope); + + node_loader_impl_exception(env, status); + /* Signal start condition */ uv_cond_signal(&node_impl->cond); @@ -3780,9 +3833,15 @@ void node_loader_impl_thread(void *data) #endif */ + // #if NODE_MAJOR_VERSION < 18 + node_loader_impl_register_linked_bindings(); + // #endif + /* Unlock node implementation mutex */ uv_mutex_unlock(&node_impl->mutex); + /* Register bindings for versions older than 18 */ + /* Start NodeJS runtime */ int result = node::Start(argc, reinterpret_cast(argv)); @@ -3824,44 +3883,6 @@ loader_impl_data node_loader_impl_initialize(loader_impl impl, configuration con (void)impl; - /* Initialize Node Loader Trampoline */ - { - static napi_module node_loader_trampoline_module = { - NAPI_MODULE_VERSION, -#if NODE_MAJOR_VERSION >= 12 - node::ModuleFlags::kLinked, -#else - 0x02, /* NM_F_LINKED */ -#endif - __FILE__, - node_loader_trampoline_initialize, - "node_loader_trampoline_module", - NULL, - { 0 } - }; - - napi_module_register(&node_loader_trampoline_module); - } - - /* Initialize Node Loader Port */ - { - static napi_module node_loader_port_module = { - NAPI_MODULE_VERSION, -#if NODE_MAJOR_VERSION >= 12 - node::ModuleFlags::kLinked, -#else - 0x02, /* NM_F_LINKED */ -#endif - __FILE__, - node_loader_port_initialize, - "node_loader_port_module", - NULL, - { 0 } - }; - - napi_module_register(&node_loader_port_module); - } - node_impl = new loader_impl_node_type(); if (node_impl == nullptr) diff --git a/source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp b/source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp index 3ce6d719d..ef20b69f4 100644 --- a/source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp +++ b/source/tests/metacall_node_python_await_extended_test/source/metacall_node_python_await_extended_test.cpp @@ -46,8 +46,11 @@ TEST_F(metacall_node_python_await_extended_test, DefaultConstructor) /* Python */ "import sys\n" "import threading\n" + "counter = 0\n" "async def python_simple(n):\n" - "\tprint('inside python_simple', threading.current_thread().ident, ':', n)\n" + "\tglobal counter\n" + "\tprint('inside python_simple', threading.current_thread().ident, counter, ':', n)\n" + "\tcounter = counter + 1\n" "\tsys.stdout.flush()\n" "\treturn n\n" // "import asyncio\n" From 30dacb44bd50e378afdae51ea9d84943826e885c Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 19:13:47 +0200 Subject: [PATCH 05/11] Checking for async ref. --- source/loaders/node_loader/source/node_loader_impl.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/loaders/node_loader/source/node_loader_impl.cpp b/source/loaders/node_loader/source/node_loader_impl.cpp index eb0150e18..a3b62c4e4 100644 --- a/source/loaders/node_loader/source/node_loader_impl.cpp +++ b/source/loaders/node_loader/source/node_loader_impl.cpp @@ -638,7 +638,10 @@ struct loader_impl_threadsafe_async_type handle_cast.async = &async_handle; - uv_close(handle_cast.handle, close_cb); + if (uv_is_active(handle_cast.handle)) + { + uv_close(handle_cast.handle, close_cb); + } } } }; From 500f2b1a3188001ab55a34f6bb1230c98feecb10 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 19:28:24 +0200 Subject: [PATCH 06/11] Trying to improve runtime builds and testing in linux. --- .github/workflows/docker-hub.yml | 4 +- .github/workflows/linux-test.yml | 2 +- tools/metacall-environment.sh | 8 ++-- tools/metacall-runtime.sh | 64 ++++++++++++++++++++++++++++++-- 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-hub.yml b/.github/workflows/docker-hub.yml index db4012ce6..12375d985 100644 --- a/.github/workflows/docker-hub.yml +++ b/.github/workflows/docker-hub.yml @@ -8,7 +8,6 @@ on: push: branches: - master - - develop tags: - 'v*.*.*' @@ -45,6 +44,9 @@ jobs: bash ./docker-compose.sh push elif [[ "${{ contains(github.ref, 'refs/tags/') }}" = true ]]; then bash ./docker-compose.sh version + else + echo "Failed to push the docker images" + exit 1 fi - name: Logout from DockerHub diff --git a/.github/workflows/linux-test.yml b/.github/workflows/linux-test.yml index 5f54ee07b..ac8bb78ce 100644 --- a/.github/workflows/linux-test.yml +++ b/.github/workflows/linux-test.yml @@ -31,7 +31,7 @@ jobs: fetch-depth: 0 - name: Install, build and run tests - run: ./docker-compose.sh test + run: ./docker-compose.sh build env: METACALL_BUILD_TYPE: ${{ matrix.build }} METACALL_BASE_IMAGE: ${{ matrix.image }} diff --git a/tools/metacall-environment.sh b/tools/metacall-environment.sh index 6f4f084f5..819a835d2 100755 --- a/tools/metacall-environment.sh +++ b/tools/metacall-environment.sh @@ -607,8 +607,8 @@ sub_c(){ case ${LINUX_DISTRO} in debian) - if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "bookworm" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then - # TODO: For now, bookworm || trixie == sid, change when bookworm || trixie is released + # For now trixie == sid, change when trixie is released + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then CODENAME="unstable" LINKNAME="" else @@ -779,8 +779,8 @@ sub_clangformat(){ case ${LINUX_DISTRO} in debian) - if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "bookworm" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then - # TODO: For now, bookworm || trixie == sid, change when bookworm || trixie is released + # For now trixie == sid, change when trixie is released + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then CODENAME="unstable" LINKNAME="" else diff --git a/tools/metacall-runtime.sh b/tools/metacall-runtime.sh index 6c1026602..5098c655a 100755 --- a/tools/metacall-runtime.sh +++ b/tools/metacall-runtime.sh @@ -202,7 +202,37 @@ sub_file(){ sub_rpc(){ echo "configure rpc" - sub_apt_install_hold libcurl4 + if [ "${OPERATIVE_SYSTEM}" = "Linux" ]; then + if [ "${LINUX_DISTRO}" = "debian" ] || [ "${LINUX_DISTRO}" = "ubuntu" ]; then + UBUNTU_CODENAME="" + CODENAME_FROM_ARGUMENTS="" + + # Obtain VERSION_CODENAME and UBUNTU_CODENAME (for Ubuntu and its derivatives) + . /etc/os-release + + case ${LINUX_DISTRO} in + debian) + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ]; then + CODENAME="unstable" + else + CODENAME="${VERSION_CODENAME}" + fi + ;; + *) + # Ubuntu and its derivatives + if [ -n "${UBUNTU_CODENAME}" ]; then + CODENAME="${UBUNTU_CODENAME}" + fi + ;; + esac + + if [ "${CODENAME}" = "trixie" ] || [ "${CODENAME}" = "unstable" ]; then + sub_apt_install_hold libcurl4t64 + else + sub_apt_install_hold libcurl4 + fi + fi + fi } # WebAssembly @@ -235,8 +265,8 @@ sub_c(){ case ${LINUX_DISTRO} in debian) - if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "bookworm" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then - # TODO: For now, bookworm || trixie == sid, change when bookworm || trixie is released + # For now trixie == sid, change when trixie is released + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then CODENAME="unstable" LINKNAME="" else @@ -293,7 +323,33 @@ sub_backtrace(){ if [ "${OPERATIVE_SYSTEM}" = "Linux" ]; then if [ "${LINUX_DISTRO}" = "debian" ] || [ "${LINUX_DISTRO}" = "ubuntu" ]; then - sub_apt_install_hold libdw1 + UBUNTU_CODENAME="" + CODENAME_FROM_ARGUMENTS="" + + # Obtain VERSION_CODENAME and UBUNTU_CODENAME (for Ubuntu and its derivatives) + . /etc/os-release + + case ${LINUX_DISTRO} in + debian) + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ]; then + CODENAME="unstable" + else + CODENAME="${VERSION_CODENAME}" + fi + ;; + *) + # Ubuntu and its derivatives + if [ -n "${UBUNTU_CODENAME}" ]; then + CODENAME="${UBUNTU_CODENAME}" + fi + ;; + esac + + if [ "${CODENAME}" = "trixie" ] || [ "${CODENAME}" = "unstable" ]; then + sub_apt_install_hold libdw1t64 libelf1t64 + else + sub_apt_install_hold libdw1 + fi elif [ "${LINUX_DISTRO}" = "alpine" ]; then $SUDO_CMD apk add --no-cache binutils fi From 956f8a82bef7be4667f68267dc9100b2fce51275 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 19:33:12 +0200 Subject: [PATCH 07/11] Solve issue with port package. --- source/ports/node_port/package-lock.json | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/source/ports/node_port/package-lock.json b/source/ports/node_port/package-lock.json index 533660323..bfb3b0d1b 100644 --- a/source/ports/node_port/package-lock.json +++ b/source/ports/node_port/package-lock.json @@ -96,12 +96,12 @@ } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" @@ -287,9 +287,9 @@ } }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "dependencies": { "to-regex-range": "^5.0.1" @@ -1010,12 +1010,12 @@ } }, "braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, "requires": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" } }, "browser-stdout": { @@ -1147,9 +1147,9 @@ "dev": true }, "fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dev": true, "requires": { "to-regex-range": "^5.0.1" From 4da180e7213f615e7b485528137a9b9443f1b264 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 19:39:00 +0200 Subject: [PATCH 08/11] Solve issues. --- tools/metacall-environment.sh | 8 ++++---- tools/metacall-runtime.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/metacall-environment.sh b/tools/metacall-environment.sh index 819a835d2..ad0315dc3 100755 --- a/tools/metacall-environment.sh +++ b/tools/metacall-environment.sh @@ -607,8 +607,8 @@ sub_c(){ case ${LINUX_DISTRO} in debian) - # For now trixie == sid, change when trixie is released - if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then + # For now bookworm || trixie == sid, change when trixie is released + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "bookworm" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then CODENAME="unstable" LINKNAME="" else @@ -779,8 +779,8 @@ sub_clangformat(){ case ${LINUX_DISTRO} in debian) - # For now trixie == sid, change when trixie is released - if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then + # For now bookworm || trixie == sid, change when trixie is released + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "bookworm" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then CODENAME="unstable" LINKNAME="" else diff --git a/tools/metacall-runtime.sh b/tools/metacall-runtime.sh index 5098c655a..77fc6e336 100755 --- a/tools/metacall-runtime.sh +++ b/tools/metacall-runtime.sh @@ -265,8 +265,8 @@ sub_c(){ case ${LINUX_DISTRO} in debian) - # For now trixie == sid, change when trixie is released - if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then + # For now bookworm || trixie == sid, change when trixie is released + if [ "${VERSION:-}" = "unstable" ] || [ "${VERSION:-}" = "testing" ] || [ "${VERSION_CODENAME}" = "bookworm" ] || [ "${VERSION_CODENAME}" = "trixie" ]; then CODENAME="unstable" LINKNAME="" else From 9565c4e25fe838b55b611eb66857a0600c059f5a Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 20:18:57 +0200 Subject: [PATCH 09/11] Solve another issue. --- source/loaders/node_loader/source/node_loader_impl.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/loaders/node_loader/source/node_loader_impl.cpp b/source/loaders/node_loader/source/node_loader_impl.cpp index a3b62c4e4..5d2f2510e 100644 --- a/source/loaders/node_loader/source/node_loader_impl.cpp +++ b/source/loaders/node_loader/source/node_loader_impl.cpp @@ -608,6 +608,9 @@ struct loader_impl_threadsafe_async_type uv_async_t async_handle; bool initialized; + loader_impl_threadsafe_async_type() : + initialized(false) {} + int initialize(loader_impl_node node_impl, void (*async_cb)(uv_async_t *)) { int result = uv_async_init(node_impl->thread_loop, &async_handle, async_cb); @@ -638,10 +641,7 @@ struct loader_impl_threadsafe_async_type handle_cast.async = &async_handle; - if (uv_is_active(handle_cast.handle)) - { - uv_close(handle_cast.handle, close_cb); - } + uv_close(handle_cast.handle, close_cb); } } }; From a3b961af64b6041ebfa92155fc0495e2becd66d2 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 20:30:09 +0200 Subject: [PATCH 10/11] Solve issue with metacallcli. --- tools/cli/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cli/Dockerfile b/tools/cli/Dockerfile index 8eaebf293..ee1eaa943 100644 --- a/tools/cli/Dockerfile +++ b/tools/cli/Dockerfile @@ -46,7 +46,7 @@ ENV LOADER_LIBRARY_PATH=/usr/local/lib \ WORKDIR $LOADER_SCRIPT_PATH # Copy cli from builder -COPY --from=builder /usr/local/bin/metacallcli /usr/local/bin/metacallcli +COPY --from=builder /usr/local/bin/metacallcli* /usr/local/bin/metacallcli # Define entry point ENTRYPOINT [ "metacallcli" ] From 991564206c8b90d87c7b693b7ff1c387209753c7 Mon Sep 17 00:00:00 2001 From: Vicente Eduardo Ferrer Garcia Date: Wed, 19 Jun 2024 23:20:23 +0200 Subject: [PATCH 11/11] Solve more bugs from runtime. --- tools/runtime/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/runtime/Dockerfile b/tools/runtime/Dockerfile index 301436b11..96e12e096 100644 --- a/tools/runtime/Dockerfile +++ b/tools/runtime/Dockerfile @@ -91,7 +91,7 @@ COPY --from=builder /usr/local/lib/plugins /usr/local/lib/plugins COPY --from=builder /usr/local/lib/node_modules/ /usr/local/lib/node_modules/ # Copy python dependencies (and port) from builder -COPY --from=builder /usr/local/lib/python3.11/dist-packages/metacall/ /usr/local/lib/python3.11/dist-packages/metacall/ +COPY --from=builder /usr/local/lib/python3.*/dist-packages/metacall/ /usr/local/lib/python3.*/dist-packages/metacall/ # Copy headers from builder COPY --from=builder /usr/local/include/metacall /usr/local/include/metacall