From e6ba25f8ce174e1e5191951e9e96dfab48ef72a2 Mon Sep 17 00:00:00 2001 From: Krishan Sharma Date: Mon, 30 Sep 2024 05:38:06 +0000 Subject: [PATCH] Includes featured off support for the MATLAB Proxy Manager. --- src/jupyter_matlab_kernel/__main__.py | 28 +- .../{kernel.py => base_kernel.py} | 416 +++++++----------- src/jupyter_matlab_kernel/jsp_kernel.py | 205 +++++++++ src/jupyter_matlab_kernel/mpm_kernel.py | 182 ++++++++ src/jupyter_matlab_kernel/test_utils.py | 55 +++ src/jupyter_matlab_proxy/__init__.py | 130 ++++-- .../unit/jupyter_matlab_kernel/test_kernel.py | 2 +- tests/unit/test_jupyter_server_proxy.py | 51 +++ 8 files changed, 772 insertions(+), 297 deletions(-) rename src/jupyter_matlab_kernel/{kernel.py => base_kernel.py} (64%) create mode 100644 src/jupyter_matlab_kernel/jsp_kernel.py create mode 100644 src/jupyter_matlab_kernel/mpm_kernel.py create mode 100644 src/jupyter_matlab_kernel/test_utils.py diff --git a/src/jupyter_matlab_kernel/__main__.py b/src/jupyter_matlab_kernel/__main__.py index ff9018b8..492f273d 100644 --- a/src/jupyter_matlab_kernel/__main__.py +++ b/src/jupyter_matlab_kernel/__main__.py @@ -1,11 +1,35 @@ # Copyright 2023-2024 The MathWorks, Inc. # Use ipykernel infrastructure to launch the MATLAB Kernel. +import os + + +def is_fallback_kernel_enabled(): + """ + Checks if the fallback kernel is enabled based on an environment variable. + + Returns: + bool: True if the fallback kernel is enabled, False otherwise. + """ + + # Get the env var toggle + use_fallback_kernel = os.getenv("MWI_USE_FALLBACK_KERNEL", "TRUE") + return use_fallback_kernel.lower().strip() == "true" + if __name__ == "__main__": from ipykernel.kernelapp import IPKernelApp from jupyter_matlab_kernel import mwi_logger - from jupyter_matlab_kernel.kernel import MATLABKernel logger = mwi_logger.get(init=True) + kernel_class = None + + if is_fallback_kernel_enabled(): + from jupyter_matlab_kernel.jsp_kernel import MATLABKernelUsingJSP + + kernel_class = MATLABKernelUsingJSP + else: + from jupyter_matlab_kernel.mpm_kernel import MATLABKernelUsingMPM + + kernel_class = MATLABKernelUsingMPM - IPKernelApp.launch_instance(kernel_class=MATLABKernel, log=logger) + IPKernelApp.launch_instance(kernel_class=kernel_class, log=logger) diff --git a/src/jupyter_matlab_kernel/kernel.py b/src/jupyter_matlab_kernel/base_kernel.py similarity index 64% rename from src/jupyter_matlab_kernel/kernel.py rename to src/jupyter_matlab_kernel/base_kernel.py index 80c756ea..213f5844 100644 --- a/src/jupyter_matlab_kernel/kernel.py +++ b/src/jupyter_matlab_kernel/base_kernel.py @@ -1,139 +1,52 @@ # Copyright 2023-2024 The MathWorks, Inc. -# Implementation of MATLAB Kernel -# Import Python Standard Library -import asyncio -import http +""" +This module serves as the base class for various MATLAB Kernels. +Examples of supported Kernels can be: +1. MATLAB Kernels that are based on Jupyter Server and Jupyter Server proxy to start +backend MATLAB proxy servers. +2. MATLAB Kernels that uses proxy manager to start backend matlab proxy servers +""" + import os import sys import time +from logging import Logger +from pathlib import Path +from typing import Optional -# Import Dependencies import aiohttp import aiohttp.client_exceptions import ipykernel.kernelbase import psutil -import requests from matlab_proxy import settings as mwi_settings from matlab_proxy import util as mwi_util -from jupyter_matlab_kernel import mwi_logger from jupyter_matlab_kernel.magic_execution_engine import ( MagicExecutionEngine, get_completion_result_for_magics, ) -from jupyter_matlab_kernel.mwi_comm_helpers import MWICommHelper from jupyter_matlab_kernel.mwi_exceptions import MATLABConnectionError _MATLAB_STARTUP_TIMEOUT = mwi_settings.get_process_startup_timeout() -_logger = mwi_logger.get() - - -def is_jupyter_testing_enabled(): - """ - Checks if testing mode is enabled - - Returns: - bool: True if MWI_JUPYTER_TEST environment variable is set to 'true' - else False - """ - - return os.environ.get("MWI_JUPYTER_TEST", "false").lower() == "true" - -def start_matlab_proxy_for_testing(logger=_logger): - """ - Only used for testing purposes. Gets the matlab-proxy server configuration - from environment variables and mocks the 'start_matlab_proxy' function - Returns: - Tuple (string, string, dict): - url (string): Complete URL to send HTTP requests to matlab-proxy - base_url (string): Complete base url for matlab-proxy obtained from tests - headers (dict): Empty dictionary +def _fetch_jupyter_base_url(parent_pid: str, logger: Logger) -> Optional[str]: """ + Fetches information about the running Jupyter server associated with the MATLAB kernel. - import matlab_proxy.util.mwi.environment_variables as mwi_env - - # These environment variables are being set by tests, using dictionary lookup - # instead of '.getenv' to make sure that the following line fails with the - # Exception 'KeyError' in case the environment variables are not set - matlab_proxy_base_url = os.environ[mwi_env.get_env_name_base_url()] - matlab_proxy_app_port = os.environ[mwi_env.get_env_name_app_port()] - - logger.debug("Creating matlab-proxy URL for MATLABKernel testing.") - - # '127.0.0.1' is used instead 'localhost' for testing since Windows machines consume - # some time to resolve 'localhost' hostname - url = "{protocol}://127.0.0.1:{port}{base_url}".format( - protocol="http", - port=matlab_proxy_app_port, - base_url=matlab_proxy_base_url, - ) - headers = {} - - logger.debug(f"matlab-proxy URL: {url}") - logger.debug(f"headers: {headers}") - - return url, matlab_proxy_base_url, headers - - -def _start_matlab_proxy_using_jupyter(url, headers, logger=_logger): - """ - Start matlab-proxy using jupyter server which started the current kernel - process by sending HTTP request to the endpoint registered through - jupyter-matlab-proxy. + This function attempts to retrieve the list of running Jupyter servers and identify the + server associated with the current MATLAB kernel based on its parent process ID. If the + Jupyter server is found, it attempts to fetch the base URL of that Jupyter Server. Args: - url (string): URL to send HTTP request - headers (dict): HTTP headers required for the request - - Returns: - bool: True if jupyter server has successfully started matlab-proxy else False. - """ - # This is content that is present in the matlab-proxy index.html page which - # can be used to validate a proper response. - matlab_proxy_index_page_identifier = "MWI_MATLAB_PROXY_IDENTIFIER" - - logger.debug( - f"Sending request to jupyter to start matlab-proxy at {url} with headers: {headers}" - ) - # send request to the matlab-proxy endpoint to make sure it is available. - # If matlab-proxy is not started, jupyter-server starts it at this point. - resp = requests.get(url, headers=headers, verify=False) - logger.debug(f"Received status code: {resp.status_code}") - - return ( - resp.status_code == http.HTTPStatus.OK - and matlab_proxy_index_page_identifier in resp.text - ) - - -def start_matlab_proxy(logger=_logger): - """ - Start matlab-proxy registered with the jupyter server which started the - current kernel process. - - Raises: - MATLABConnectionError: Occurs when kernel is not started by jupyter server. + parent_pid: process ID (PID) of the Kernel's parent process. + logger (Logger): The logger instance for logging debug information. Returns: - Tuple (string, string, dict): - url (string): Complete URL to send HTTP requests to matlab-proxy - base_url (string): Complete base url for matlab-proxy provided by jupyter server - headers (dict): HTTP headers required while sending HTTP requests to matlab-proxy + base_url (str): The base URL of the Jupyter server, if found. """ - - # If jupyter testing is enabled, then a standalone matlab-proxy server would be - # launched by the tests and kernel would expect the configurations of this matlab-proxy - # server which is provided through environment variables to 'start_matlab_proxy_for_testing' - if is_jupyter_testing_enabled(): - return start_matlab_proxy_for_testing(logger) - nb_server_list = [] - - # The matlab-proxy server, if running, could have been started by either - # "jupyter_server" or "notebook" package. try: from jupyter_server import serverapp @@ -145,92 +58,46 @@ def start_matlab_proxy(logger=_logger): except ImportError: pass - # Use parent process id of the kernel to filter Jupyter Server from the list. - jupyter_server_pid = os.getppid() - - # On Windows platforms using venv/virtualenv an intermediate python process spaws the kernel. - # jupyter_server ---spawns---> intermediate_process ---spawns---> jupyter_matlab_kernel - # Thus we need to go one level higher to acquire the process id of the jupyter server. - # Note: conda environments do not require this, and for these environments sys.prefix == sys.base_prefix - is_virtual_env = sys.prefix != sys.base_prefix - if mwi_util.system.is_windows() and is_virtual_env: - jupyter_server_pid = psutil.Process(jupyter_server_pid).ppid() - - logger.debug(f"Resolved jupyter server pid: {jupyter_server_pid}") - - nb_server = dict() + nb_server = {} found_nb_server = False for server in nb_server_list: - if server["pid"] == jupyter_server_pid: - logger.debug("Jupyter server associated with this MATLAB Kernel found.") + if server["pid"] == parent_pid: found_nb_server = True nb_server = server # Stop iterating over the server list - break + return nb_server["base_url"] - # Error out if the server is not found! + # log and return None if the server is not found if not found_nb_server: - logger.error("Jupyter server associated with this MATLABKernel not found.") - raise MATLABConnectionError( - """ - Error: MATLAB Kernel for Jupyter was unable to find the notebook server from which it was spawned!\n - Resolution: Please relaunch kernel from JupyterLab or Classic Jupyter Notebook. - """ + logger.debug( + "Jupyter server associated with this MATLAB Kernel not found, might a non-jupyter based MATLAB Kernel" ) + return None - # Verify that Password is disabled - if nb_server["password"] is True: - logger.error("Jupyter server uses password for authentication.") - # TODO: To support passwords, we either need to acquire it from Jupyter or ask the user? - raise MATLABConnectionError( - """ - Error: MATLAB Kernel could not communicate with MATLAB.\n - Reason: There is a password set to access the Jupyter server.\n - Resolution: Delete the cached Notebook password file, and restart the kernel.\n - See https://jupyter-notebook.readthedocs.io/en/stable/public_server.html#securing-a-notebook-server for more information. - """ - ) - # Using nb_server["url"] to construct matlab-proxy URL as it handles the following cases - # 1. For normal usage of Jupyter, the URL returned by nb_server uses localhost - # 2. For explicitly specified IP with Jupyter, the URL returned by nb_server - # a. uses FQDN hostname when specified IP is 0.0.0.0 - # b. uses specified IP for all other cases - matlab_proxy_url = "{jupyter_server_url}matlab".format( - jupyter_server_url=nb_server["url"] - ) - - available_tokens = { - "jupyter_server": nb_server.get("token"), - "jupyterhub": os.getenv("JUPYTERHUB_API_TOKEN"), - "default": None, - } +def _get_parent_pid() -> str: + """ + Retrieves the parent process ID (PID) of the Kernel process. - for token in available_tokens.values(): - if token: - headers = {"Authorization": f"token {token}"} - else: - headers = None + This function determines the process ID of the parent (Jupyter/VSCode) that spawned the + current kernel. On Windows platforms using virtual environments, it accounts for an + intermediate process that may spawn the kernel, by going one level higher in the process + hierarchy to obtain the correct parent PID. - if _start_matlab_proxy_using_jupyter(matlab_proxy_url, headers, logger): - logger.debug( - f"Started matlab-proxy using jupyter at {matlab_proxy_url} with headers: {headers}" - ) - return matlab_proxy_url, nb_server["base_url"], headers + Returns: + str: The PID of the Jupyter server. + """ + parent_pid = os.getppid() - logger.error( - "MATLABKernel could not communicate with matlab-proxy through Jupyter server" - ) - logger.error(f"Jupyter server:\n{nb_server}") - raise MATLABConnectionError( - """ - Error: MATLAB Kernel could not communicate with MATLAB. - Reason: Possibly due to invalid jupyter security tokens. - """ - ) + # Note: conda environments do not require this, and for these environments + # sys.prefix == sys.base_prefix + is_virtual_env = sys.prefix != sys.base_prefix + if mwi_util.system.is_windows() and is_virtual_env: + parent_pid = psutil.Process(parent_pid).ppid() + return parent_pid -class MATLABKernel(ipykernel.kernelbase.Kernel): +class BaseMATLABKernel(ipykernel.kernelbase.Kernel): # Required variables for Jupyter Kernel to function # banner is shown only for Jupyter Console. banner = "MATLAB" @@ -244,40 +111,35 @@ class MATLABKernel(ipykernel.kernelbase.Kernel): "file_extension": ".m", } - # MATLAB Kernel state - kernel_id = "" - server_base_url = "" - startup_error = None - startup_checks_completed: bool = False - def __init__(self, *args, **kwargs): # Call superclass constructor to initialize ipykernel infrastructure - super(MATLABKernel, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) + + # Kernel identifier which is used to setup loggers as well as to track the + # mapping between this kernel and the backend MATLAB proxy process when proxy-manager + # is used as the MATLAB proxy provisioner. + self.kernel_id = self._extract_kernel_id_from_sys_args(sys.argv) + + # Provides the base_url for the matlab-proxy which is assigned to this Kernel instance + self.matlab_proxy_base_url = "" + + # Used to track if there was any errors during MATLAB proxy startup + self.startup_error = None + + # base_url field for Jupyter Server, required for performing licensing + self.jupyter_base_url = None + + # Keeps track of whether the startup checks were completed or not + self.startup_checks_completed: bool = False - # Update log instance with kernel id. This helps in identifying logs from - # multiple kernels which are running simultaneously - self.kernel_id = self.ident self.log.debug(f"Initializing kernel with id: {self.kernel_id}") self.log = self.log.getChild(f"{self.kernel_id}") # Initialize the Magic Execution Engine. self.magic_engine = MagicExecutionEngine(self.log) - try: - # Start matlab-proxy using the jupyter-matlab-proxy registered endpoint. - murl, self.server_base_url, headers = start_matlab_proxy(self.log) - - # Using asyncio.get_event_loop for shell_loop as io_loop variable is - # not yet initialized because start() is called after the __init__ - # is completed. - shell_loop = asyncio.get_event_loop() - control_loop = self.control_thread.io_loop.asyncio_loop - self.mwi_comm_helper = MWICommHelper( - self.kernel_id, murl, shell_loop, control_loop, headers, self.log - ) - shell_loop.run_until_complete(self.mwi_comm_helper.connect()) - except MATLABConnectionError as err: - self.startup_error = err + # Communication helper for interaction with backend MATLAB proxy + self.mwi_comm_helper = None # ipykernel Interface API # https://ipython.readthedocs.io/en/stable/development/wrapperkernels.html @@ -540,34 +402,56 @@ async def do_history( hist_access_type, output, raw, session, start, stop, n, pattern, unique ) - async def do_shutdown(self, restart): - self.log.debug("Received shutdown request from Jupyter") - try: - await self.mwi_comm_helper.send_shutdown_request_to_matlab() - await self.mwi_comm_helper.disconnect() - except ( - MATLABConnectionError, - aiohttp.client_exceptions.ClientResponseError, - ) as e: - self.log.error( - f"Exception occurred while sending shutdown request to MATLAB:\n{e}" - ) + # Helper functions - return super().do_shutdown(restart) + def display_output(self, out): + """ + Common function to send execution outputs to Jupyter UI. + For more information, look at https://jupyter-client.readthedocs.io/en/stable/messaging.html#messages-on-the-iopub-pub-sub-channel - # Helper functions + Input Example: + 1. Execution Output: + out = { + "type": "execute_result", + "mimetype": ["text/plain","text/html"], + "value": ["Hello","Hello"] + } + 2. For all other message types: + out = { + "type": "stream", + "content": { + "name": "stderr", + "text": "An error occurred" + } + } + + Args: + out (dict): A dictionary containing the type of output and the content of the output. + """ + msg_type = out["type"] + if msg_type == "execute_result": + assert len(out["mimetype"]) == len(out["value"]) + response = { + # Use zip to create a tuple of KV pair of mimetype and value. + "data": dict(zip(out["mimetype"], out["value"])), + "metadata": {}, + "execution_count": self.execution_count, + } + else: + response = out["content"] + self.send_response(self.iopub_socket, msg_type, response) - async def perform_startup_checks(self): + async def perform_startup_checks(self, iframe_src: str = None): """ One time checks triggered during the first execution request. Displays login window if matlab is not licensed using matlab-proxy. Raises: - ClientError, MATLABConnectionError: Occurs when matlab-proxy is not started or kernel cannot - communicate with MATLAB. + ClientError, MATLABConnectionError: Occurs when matlab-proxy is not started or + kernel cannot communicate with MATLAB. """ self.log.debug("Performing startup checks") - # Incase an error occurred while kernel initialization, display it to the user. + # In case an error occurred while kernel initialization, display it to the user. if self.startup_error is not None: self.log.error(f"Found a startup error: {self.startup_error}") raise self.startup_error @@ -592,12 +476,13 @@ async def perform_startup_checks(self): self.log.debug( "MATLAB is not licensed. Displaying HTML output to enable licensing." ) + self.log.debug(f"{iframe_src=}") self.display_output( { "type": "display_data", "content": { "data": { - "text/html": f'' + "text/html": f'' }, "metadata": {}, }, @@ -605,6 +490,28 @@ async def perform_startup_checks(self): ) # Wait until MATLAB is started before sending requests. + await self.poll_for_matlab_startup( + is_matlab_licensed, matlab_status, matlab_proxy_has_error + ) + + async def poll_for_matlab_startup( + self, is_matlab_licensed, matlab_status, matlab_proxy_has_error + ): + """Wait until MATLAB has started or time has run out" + + Args: + is_matlab_licensed (bool): A flag indicating whether MATLAB is + licensed and eligible to start. + matlab_status (str): A string representing the current status + of the MATLAB startup process. + matlab_proxy_has_error (bool): A flag indicating whether there + is an error in the MATLAB proxy process during startup. + + Raises: + MATLABConnectionError: If an error occurs while attempting to + connect to the MATLAB backend, or if MATLAB fails to start + within the expected timeframe. + """ self.log.debug("Waiting until MATLAB is started") timeout = 0 while ( @@ -650,39 +557,44 @@ async def perform_startup_checks(self): self.log.debug("MATLAB is running, startup checks completed.") - def display_output(self, out): + def _extract_kernel_id_from_sys_args(self, args) -> str: """ - Common function to send execution outputs to Jupyter UI. - For more information, look at https://jupyter-client.readthedocs.io/en/stable/messaging.html#messages-on-the-iopub-pub-sub-channel + Extracts the kernel ID from the system arguments. - Input Example: - 1. Execution Output: - out = { - "type": "execute_result", - "mimetype": ["text/plain","text/html"], - "value": ["Hello","Hello"] - } - 2. For all other message types: - out = { - "type": "stream", - "content": { - "name": "stderr", - "text": "An error occurred" - } - } + This function parses the system arguments to extract the kernel ID from the Jupyter + kernel connection file path. The expected format of the arguments is: + ['/path/to/jupyter_matlab_kernel/__main__.py', '-f', '/path/to/kernel-.json']. + If the extraction fails, it logs a debug message and returns another identifier (self.ident) + from the kernel base class. Args: - out (dict): A dictionary containing the type of output and the content of the output. + args (list): The list of system arguments. + + Returns: + str: The extracted kernel ID if successful, otherwise `self.ident`. + + Notes: + self.ident is another random UUID and is not as same as kernel id which we are using + for logs correlation as well as mapping the backend MATLAB proxy to a MATLAB Kernel + at proxy manager layer. Users will not be able to route to their corresponding MATLAB + when they click "Open MATLAB" button from their notebook interface, for isolated MATLAB. + As of now, connection file name is the only source of truth that supplies the kernel id + correctly. The issue of not being able to route to corresponding MATLAB is only specific + to Jupyter (via Jupyter Server Proxy) and specifically in isolated MATLAB workflows and + won't have impact on VSCode or other clients (e.g. test client). + """ - msg_type = out["type"] - if msg_type == "execute_result": - assert len(out["mimetype"]) == len(out["value"]) - response = { - # Use zip to create a tuple of KV pair of mimetype and value. - "data": dict(zip(out["mimetype"], out["value"])), - "metadata": {}, - "execution_count": self.execution_count, - } - else: - response = out["content"] - self.send_response(self.iopub_socket, msg_type, response) + try: + connection_file_path: Path = Path(args[2]) + + # Get the final component of the path without the suffix + kernel_file_name = connection_file_path.stem + + # Jupyter kernel connection file naming scheme -> kernel-a8623c0a-574e-4f3d-a03a-ccb8c3f21165.json + # VSCode kernel connection file naming scheme -> kernel-v2-3030095VYYhRYlRs0Eu.json + return kernel_file_name.split("kernel-")[1] + except Exception as e: + self.log.debug( + f"Unable to extract kernel id from the sys args with ex: {e}" + ) + return self.ident diff --git a/src/jupyter_matlab_kernel/jsp_kernel.py b/src/jupyter_matlab_kernel/jsp_kernel.py new file mode 100644 index 00000000..876b4590 --- /dev/null +++ b/src/jupyter_matlab_kernel/jsp_kernel.py @@ -0,0 +1,205 @@ +# Copyright 2024 The MathWorks, Inc. + +"""This module contains derived class implementation of MATLABKernel that uses +Jupyter Server to manage interactions with matlab-proxy & MATLAB. +""" + +import asyncio +import http +import os + +# Import Dependencies +import aiohttp +import aiohttp.client_exceptions +import requests + +from jupyter_matlab_kernel import base_kernel as base +from jupyter_matlab_kernel import mwi_logger, test_utils +from jupyter_matlab_kernel.mwi_comm_helpers import MWICommHelper +from jupyter_matlab_kernel.mwi_exceptions import MATLABConnectionError + +_logger = mwi_logger.get() + + +def _start_matlab_proxy_using_jupyter(url, headers, logger=_logger): + """ + Start matlab-proxy using jupyter server which started the current kernel + process by sending HTTP request to the endpoint registered through + jupyter-matlab-proxy. + + Args: + url (string): URL to send HTTP request + headers (dict): HTTP headers required for the request + + Returns: + bool: True if jupyter server has successfully started matlab-proxy else False. + """ + # This is content that is present in the matlab-proxy index.html page which + # can be used to validate a proper response. + matlab_proxy_index_page_identifier = "MWI_MATLAB_PROXY_IDENTIFIER" + + logger.debug( + f"Sending request to jupyter to start matlab-proxy at {url} with headers: {headers}" + ) + # send request to the matlab-proxy endpoint to make sure it is available. + # If matlab-proxy is not started, jupyter-server starts it at this point. + resp = requests.get(url, headers=headers, verify=False) + logger.debug("Received status code: %s", resp.status_code) + + return ( + resp.status_code == http.HTTPStatus.OK + and matlab_proxy_index_page_identifier in resp.text + ) + + +def start_matlab_proxy(logger=_logger): + """ + Start matlab-proxy registered with the jupyter server which started the + current kernel process. + + Raises: + MATLABConnectionError: Occurs when kernel is not started by jupyter server. + + Returns: + Tuple (string, string, dict): + url (string): Complete URL to send HTTP requests to matlab-proxy + base_url (string): Complete base url for matlab-proxy provided by jupyter server + headers (dict): HTTP headers required while sending HTTP requests to matlab-proxy + """ + + # If jupyter testing is enabled, then a standalone matlab-proxy server would be + # launched by the tests and kernel would expect the configurations of this matlab-proxy + # server which is provided through environment variables to 'start_matlab_proxy_for_testing' + if test_utils.is_jupyter_testing_enabled(): + return test_utils.start_matlab_proxy_for_testing(logger) + + nb_server_list = [] + + # The matlab-proxy server, if running, could have been started by either + # "jupyter_server" or "notebook" package. + try: + from jupyter_server import serverapp + + nb_server_list += list(serverapp.list_running_servers()) + + from notebook import notebookapp + + nb_server_list += list(notebookapp.list_running_servers()) + except ImportError: + pass + + # Use parent process id of the kernel to filter Jupyter Server from the list. + jupyter_server_pid = base._get_parent_pid() + logger.debug(f"Resolved jupyter server pid: {jupyter_server_pid}") + + nb_server = dict() + found_nb_server = False + for server in nb_server_list: + if server["pid"] == jupyter_server_pid: + logger.debug("Jupyter server associated with this MATLAB Kernel found.") + found_nb_server = True + nb_server = server + # Stop iterating over the server list + break + + # Error out if the server is not found! + if not found_nb_server: + logger.error("Jupyter server associated with this MATLABKernel not found.") + raise MATLABConnectionError( + """ + Error: MATLAB Kernel for Jupyter was unable to find the notebook server from which it was spawned!\n + Resolution: Please relaunch kernel from JupyterLab or Classic Jupyter Notebook. + """ + ) + + # Verify that Password is disabled + if nb_server["password"] is True: + logger.error("Jupyter server uses password for authentication.") + # TODO: To support passwords, we either need to acquire it from Jupyter or ask the user? + raise MATLABConnectionError( + """ + Error: MATLAB Kernel could not communicate with MATLAB.\n + Reason: There is a password set to access the Jupyter server.\n + Resolution: Delete the cached Notebook password file, and restart the kernel.\n + See https://jupyter-notebook.readthedocs.io/en/stable/public_server.html#securing-a-notebook-server for more information. + """ + ) + + # Using nb_server["url"] to construct matlab-proxy URL as it handles the following cases + # 1. For normal usage of Jupyter, the URL returned by nb_server uses localhost + # 2. For explicitly specified IP with Jupyter, the URL returned by nb_server + # a. uses FQDN hostname when specified IP is 0.0.0.0 + # b. uses specified IP for all other cases + matlab_proxy_url = "{jupyter_server_url}matlab".format( + jupyter_server_url=nb_server["url"] + ) + + available_tokens = { + "jupyter_server": nb_server.get("token"), + "jupyterhub": os.getenv("JUPYTERHUB_API_TOKEN"), + "default": None, + } + + for token in available_tokens.values(): + if token: + headers = {"Authorization": f"token {token}"} + else: + headers = None + + if _start_matlab_proxy_using_jupyter(matlab_proxy_url, headers, logger): + logger.debug( + f"Started matlab-proxy using jupyter at {matlab_proxy_url} with headers: {headers}" + ) + return matlab_proxy_url, nb_server["base_url"], headers + + logger.error( + "MATLABKernel could not communicate with matlab-proxy through Jupyter server" + ) + logger.error(f"Jupyter server:\n{nb_server}") + raise MATLABConnectionError( + """ + Error: MATLAB Kernel could not communicate with MATLAB. + Reason: Possibly due to invalid jupyter security tokens. + """ + ) + + +class MATLABKernelUsingJSP(base.BaseMATLABKernel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + try: + # Start matlab-proxy using the jupyter-matlab-proxy registered endpoint. + murl, self.jupyter_base_url, headers = start_matlab_proxy(self.log) + + # Using asyncio.get_event_loop for shell_loop as io_loop variable is + # not yet initialized because start() is called after the __init__ + # is completed. + shell_loop = asyncio.get_event_loop() + control_loop = self.control_thread.io_loop.asyncio_loop + self.mwi_comm_helper = MWICommHelper( + self.kernel_id, murl, shell_loop, control_loop, headers, self.log + ) + shell_loop.run_until_complete(self.mwi_comm_helper.connect()) + except MATLABConnectionError as err: + self.startup_error = err + + async def do_shutdown(self, restart): + self.log.debug("Received shutdown request from Jupyter") + try: + await self.mwi_comm_helper.send_shutdown_request_to_matlab() + await self.mwi_comm_helper.disconnect() + except ( + MATLABConnectionError, + aiohttp.client_exceptions.ClientResponseError, + ) as e: + self.log.error( + f"Exception occurred while sending shutdown request to MATLAB:\n{e}" + ) + + return super().do_shutdown(restart) + + async def perform_startup_checks(self): + """Overriding base function to provide a different iframe source""" + iframe_src: str = f'{self.jupyter_base_url + "matlab"}' + await super().perform_startup_checks(iframe_src) diff --git a/src/jupyter_matlab_kernel/mpm_kernel.py b/src/jupyter_matlab_kernel/mpm_kernel.py new file mode 100644 index 00000000..7c1b3013 --- /dev/null +++ b/src/jupyter_matlab_kernel/mpm_kernel.py @@ -0,0 +1,182 @@ +# Copyright 2024 The MathWorks, Inc. + +"""This module contains derived class implementation of MATLABKernel that uses +MATLAB Proxy Manager to manage interactions with matlab-proxy & MATLAB. +""" + +from logging import Logger + +import matlab_proxy_manager.lib.api as mpm_lib +from requests.exceptions import HTTPError + +from jupyter_matlab_kernel import base_kernel as base +from jupyter_matlab_kernel.mwi_comm_helpers import MWICommHelper +from jupyter_matlab_kernel.mwi_exceptions import MATLABConnectionError + + +class MATLABKernelUsingMPM(base.BaseMATLABKernel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + # Used to detect if this Kernel has been assigned a MATLAB-proxy server or not + self.is_matlab_assigned = False + + # Serves as the auth token to secure communication between Jupyter Server and MATLAB proxy manager + self.mpm_auth_token = None + + # There might be multiple instances of Jupyter servers or VScode servers running on a + # single machine. This attribute serves as the context provider and backend MATLAB proxy + # processes are filtered using this attribute during start and shutdown of MATLAB proxy + self.parent_pid = base._get_parent_pid() + + # Required for performing licensing using Jupyter Server + self.jupyter_base_url = base._fetch_jupyter_base_url(self.parent_pid, self.log) + + # ipykernel Interface API + # https://ipython.readthedocs.io/en/stable/development/wrapperkernels.html + + async def do_execute( + self, + code, + silent, + store_history=True, + user_expressions=None, + allow_stdin=False, + *, + cell_id=None, + ): + """ + Used by ipykernel infrastructure for execution. For more info, look at + https://jupyter-client.readthedocs.io/en/stable/messaging.html#execute + """ + self.log.debug(f"Received execution request from Jupyter with code:\n{code}") + + # Starts the matlab proxy process if this kernel hasn't yet been assigned a + # matlab proxy and sets the attributes on kernel to talk to the correct backend. + if not self.is_matlab_assigned: + self.log.debug("Starting matlab-proxy") + await self._start_matlab_proxy_and_comm_helper() + self.is_matlab_assigned = True + + return await super().do_execute( + code=code, + silent=silent, + store_history=store_history, + user_expressions=user_expressions, + allow_stdin=allow_stdin, + cell_id=cell_id, + ) + + async def do_shutdown(self, restart): + self.log.debug("Received shutdown request from Jupyter") + if self.is_matlab_assigned: + try: + # Cleans up internal live editor state, client session + await self.mwi_comm_helper.send_shutdown_request_to_matlab() + await self.mwi_comm_helper.disconnect() + + except (MATLABConnectionError, HTTPError) as e: + self.log.error( + f"Exception occurred while sending shutdown request to MATLAB:\n{e}" + ) + except Exception as e: + self.log.debug("Exception during shutdown", e) + finally: + # Shuts down matlab assigned to this Kernel (based on satisfying certain criteria) + await mpm_lib.shutdown( + self.parent_pid, self.kernel_id, self.mpm_auth_token + ) + self.is_matlab_assigned = False + + return super().do_shutdown(restart) + + async def perform_startup_checks(self): + """Overriding base function to provide a different iframe source""" + iframe_src: str = ( + f'{self.jupyter_base_url}{self.matlab_proxy_base_url.lstrip("/")}/' + ) + await super().perform_startup_checks(iframe_src) + + # Helper functions + + async def _start_matlab_proxy_and_comm_helper(self) -> None: + """ + Starts the MATLAB proxy using the proxy manager and fetches its status. + """ + try: + ( + murl, + self.matlab_proxy_base_url, + headers, + self.mpm_auth_token, + ) = await self._initialize_matlab_proxy_with_mpm(self.log) + + await self._initialize_mwi_comm_helper(murl, headers) + except MATLABConnectionError as err: + self.startup_error = err + + async def _initialize_matlab_proxy_with_mpm(self, _logger: Logger): + """ + Initializes the MATLAB proxy process using the Proxy Manager (MPM) library. + + Calls proxy manager to start the MATLAB proxy process for this kernel. + + Args: + logger (Logger): The logger instance + + Returns: + tuple: A tuple containing: + - server_url (str): Absolute URL for the MATLAB proxy backend (includes base URL) + - base_url (str): The base URL of the MATLAB proxy server + - headers (dict): The headers required for communication with the MATLAB proxy + - mpm_auth_token (str): Token for authentication between kernel and proxy manager + + Raises: + MATLABConnectionError: If the MATLAB proxy process could not be started + """ + try: + response = await mpm_lib.start_matlab_proxy_for_kernel( + caller_id=self.kernel_id, + parent_id=self.parent_pid, + is_isolated_matlab=False, + ) + return ( + response.get("absolute_url"), + response.get("mwi_base_url"), + response.get("headers"), + response.get("mpm_auth_token"), + ) + except Exception as e: + _logger.error( + f"MATLAB Kernel could not start matlab-proxy using proxy manager with error: {e}" + ) + raise MATLABConnectionError( + """ + Error: MATLAB Kernel could not start the MATLAB proxy process via proxy manager. + """ + ) from e + + async def _initialize_mwi_comm_helper(self, murl, headers): + """ + Initializes the MWICommHelper for managing communication with a specified URL. + + This method sets up the MWICommHelper instance with the given + message URL and headers, utilizing the shell and control event loops. It then + initiates a connection by creating and awaiting a task on the shell event loop. + + Parameters: + - murl (str): The message URL used for communication. + - headers (dict): A dictionary of headers to include in the communication setup. + """ + shell_loop = self.io_loop.asyncio_loop + control_loop = self.control_thread.io_loop.asyncio_loop + self.mwi_comm_helper = MWICommHelper( + self.kernel_id, murl, shell_loop, control_loop, headers, self.log + ) + await self.mwi_comm_helper.connect() + + def _process_children(self): + """Overrides the _process_children in kernelbase class to not return the list of children + so that the child process termination can be managed at proxy manager layer + """ + return [] diff --git a/src/jupyter_matlab_kernel/test_utils.py b/src/jupyter_matlab_kernel/test_utils.py new file mode 100644 index 00000000..89e42f03 --- /dev/null +++ b/src/jupyter_matlab_kernel/test_utils.py @@ -0,0 +1,55 @@ +# Copyright 2023-2024 The MathWorks, Inc. + +import os +from jupyter_matlab_kernel import mwi_logger + +_logger = mwi_logger.get() + + +def is_jupyter_testing_enabled(): + """ + Checks if testing mode is enabled + + Returns: + bool: True if MWI_JUPYTER_TEST environment variable is set to 'true' + else False + """ + + return os.environ.get("MWI_JUPYTER_TEST", "false").lower() == "true" + + +def start_matlab_proxy_for_testing(logger=_logger): + """ + Only used for testing purposes. Gets the matlab-proxy server configuration + from environment variables and mocks the 'start_matlab_proxy' function + + Returns: + Tuple (string, string, dict): + url (string): Complete URL to send HTTP requests to matlab-proxy + base_url (string): Complete base url for matlab-proxy obtained from tests + headers (dict): Empty dictionary + """ + + import matlab_proxy.util.mwi.environment_variables as mwi_env + + # These environment variables are being set by tests, using dictionary lookup + # instead of '.getenv' to make sure that the following line fails with the + # Exception 'KeyError' in case the environment variables are not set + matlab_proxy_base_url = os.environ[mwi_env.get_env_name_base_url()] + matlab_proxy_app_port = os.environ[mwi_env.get_env_name_app_port()] + + logger.debug("Creating matlab-proxy URL for MATLABKernel testing.") + + # '127.0.0.1' is used instead 'localhost' for testing since Windows machines consume + # some time to resolve 'localhost' hostname + url = "{protocol}://127.0.0.1:{port}{base_url}".format( + protocol="http", + port=matlab_proxy_app_port, + base_url=matlab_proxy_base_url, + ) + headers = {} + + logger.debug("matlab-proxy URL: %s", url) + logger.debug("headers: %s", headers) + + return url, matlab_proxy_base_url, headers diff --git a/src/jupyter_matlab_proxy/__init__.py b/src/jupyter_matlab_proxy/__init__.py index 455936ba..ef639e51 100644 --- a/src/jupyter_matlab_proxy/__init__.py +++ b/src/jupyter_matlab_proxy/__init__.py @@ -1,13 +1,23 @@ # Copyright 2020-2024 The MathWorks, Inc. import os +import secrets from pathlib import Path +import matlab_proxy +from matlab_proxy.constants import MWI_AUTH_TOKEN_NAME_FOR_HTTP from matlab_proxy.util.mwi import environment_variables as mwi_env +from matlab_proxy.util.mwi import logger as mwi_logger from matlab_proxy.util.mwi import token_auth as mwi_token_auth from jupyter_matlab_proxy.jupyter_config import config +_MPM_AUTH_TOKEN: str = secrets.token_hex(32) +_JUPYTER_SERVER_PID: str = str(os.getpid()) +_USE_FALLBACK_KERNEL: bool = ( + os.getenv("MWI_USE_FALLBACK_KERNEL", "TRUE").lower().strip() == "true" +) + def _get_auth_token(): """ @@ -48,24 +58,34 @@ def _get_env(port, base_url): Returns: [Dict]: Containing environment settings to launch the MATLAB Desktop. """ + env = {} + if _USE_FALLBACK_KERNEL: + env = { + mwi_env.get_env_name_app_port(): str(port), + mwi_env.get_env_name_base_url(): f"{base_url}matlab", + mwi_env.get_env_name_app_host(): "127.0.0.1", + } - env = { - mwi_env.get_env_name_app_port(): str(port), - mwi_env.get_env_name_base_url(): f"{base_url}matlab", - mwi_env.get_env_name_app_host(): "127.0.0.1", - } - - # Add token authentication related information to the environment variables - # dictionary passed to the matlab-proxy process if token authentication is - # not explicitly disabled. - if _mwi_auth_token: - env.update( - { - mwi_env.get_env_name_enable_mwi_auth_token(): "True", - mwi_env.get_env_name_mwi_auth_token(): _mwi_auth_token.get("token"), - } - ) - + # Add token authentication related information to the environment variables + # dictionary passed to the matlab-proxy process if token authentication is + # not explicitly disabled. + if _mwi_auth_token: + env.update( + { + mwi_env.get_env_name_enable_mwi_auth_token(): "True", + mwi_env.get_env_name_mwi_auth_token(): _mwi_auth_token.get("token"), + } + ) + + else: + # case when we are using matlab proxy manager + import matlab_proxy_manager.utils.environment_variables as mpm_env + + env = { + mpm_env.get_env_name_mwi_mpm_port(): str(port), + mpm_env.get_env_name_mwi_mpm_auth_token(): _MPM_AUTH_TOKEN, + mpm_env.get_env_name_mwi_mpm_parent_pid(): _JUPYTER_SERVER_PID, + } return env @@ -76,36 +96,62 @@ def setup_matlab(): [Dict]: Containing information to launch the MATLAB Desktop. """ - import matlab_proxy - from matlab_proxy.constants import MWI_AUTH_TOKEN_NAME_FOR_HTTP - from matlab_proxy.util.mwi import logger as mwi_logger - logger = mwi_logger.get(init=True) logger.info("Initializing Jupyter MATLAB Proxy") + jsp_config = _get_jsp_config(logger=logger) + + return jsp_config + + +def _get_jsp_config(logger): icon_path = Path(__file__).parent / "icon_open_matlab.svg" - logger.debug(f"Icon_path: {icon_path}") - logger.debug(f"Launch Command: {matlab_proxy.get_executable_name()}") - logger.debug(f"Extension Name: {config['extension_name']}") - - jsp_config = { - "command": [ - matlab_proxy.get_executable_name(), - "--config", - config["extension_name"], - ], - "timeout": 100, - "environment": _get_env, - "absolute_url": True, - "launcher_entry": {"title": "Open MATLAB", "icon_path": icon_path}, - } - - # Add token_hash information to the request_headers_override option to - # ensure requests from jupyter to matlab-proxy are automatically authenticated. - # We are using token_hash instead of raw token for better security. - if _mwi_auth_token: + logger.debug("Icon_path: %s", icon_path) + jsp_config = {} + + if _USE_FALLBACK_KERNEL: + jsp_config = { + "command": [ + matlab_proxy.get_executable_name(), + "--config", + config["extension_name"], + ], + "timeout": 100, + "environment": _get_env, + "absolute_url": True, + "launcher_entry": {"title": "Open MATLAB", "icon_path": icon_path}, + } + logger.debug("Launch Command: %s", jsp_config.get("command")) + + # Add token_hash information to the request_headers_override option to + # ensure requests from jupyter to matlab-proxy are automatically authenticated. + # We are using token_hash instead of raw token for better security. + if _mwi_auth_token: + jsp_config["request_headers_override"] = { + MWI_AUTH_TOKEN_NAME_FOR_HTTP: _mwi_auth_token.get("token_hash") + } + else: + import matlab_proxy_manager + from matlab_proxy_manager.utils import constants + + # JSP config for when we are using matlab proxy manager + jsp_config = { + # Starts proxy manager process which in turn starts a shared matlab proxy instance + # if not already started. This gets invoked on clicking `Open MATLAB` button and would + # always take the user to the default (shared) matlab-proxy instance. + "command": [matlab_proxy_manager.get_executable_name()], + "timeout": 100, # timeout in seconds + "environment": _get_env, + "absolute_url": True, + "launcher_entry": {"title": "Open MATLAB", "icon_path": icon_path}, + } + logger.debug("Launch Command: %s", jsp_config.get("command")) + + # Add jupyter server pid and mpm_auth_token to the request headers for resource + # filtering and Jupyter to proxy manager authentication jsp_config["request_headers_override"] = { - MWI_AUTH_TOKEN_NAME_FOR_HTTP: _mwi_auth_token.get("token_hash") + constants.HEADER_MWI_MPM_CONTEXT: _JUPYTER_SERVER_PID, + constants.HEADER_MWI_MPM_AUTH_TOKEN: _MPM_AUTH_TOKEN, } return jsp_config diff --git a/tests/unit/jupyter_matlab_kernel/test_kernel.py b/tests/unit/jupyter_matlab_kernel/test_kernel.py index 09a4b931..e433f357 100644 --- a/tests/unit/jupyter_matlab_kernel/test_kernel.py +++ b/tests/unit/jupyter_matlab_kernel/test_kernel.py @@ -3,7 +3,7 @@ # This file contains tests for jupyter_matlab_kernel.kernel import mocks.mock_jupyter_server as MockJupyterServer import pytest -from jupyter_matlab_kernel.kernel import start_matlab_proxy +from jupyter_matlab_kernel.jsp_kernel import start_matlab_proxy from jupyter_matlab_kernel.mwi_exceptions import MATLABConnectionError from jupyter_server import serverapp from mocks.mock_jupyter_server import MockJupyterServerFixture diff --git a/tests/unit/test_jupyter_server_proxy.py b/tests/unit/test_jupyter_server_proxy.py index 2eee99ed..600ca8cc 100644 --- a/tests/unit/test_jupyter_server_proxy.py +++ b/tests/unit/test_jupyter_server_proxy.py @@ -6,9 +6,12 @@ import jupyter_matlab_proxy import matlab_proxy +import matlab_proxy_manager from jupyter_matlab_proxy.jupyter_config import config from matlab_proxy.constants import MWI_AUTH_TOKEN_NAME_FOR_HTTP from matlab_proxy.util.mwi import environment_variables as mwi_env +from matlab_proxy_manager.utils import constants +from matlab_proxy_manager.utils import environment_variables as mpm_env def test_get_auth_token(): @@ -55,6 +58,19 @@ def test_get_env_with_token_auth_disabled(monkeypatch): assert r.get(mwi_env.get_env_name_mwi_auth_token()) == None +def test_get_env_with_proxy_manager(monkeypatch): + """Tests if _get_env() method returns the expected environment settings as a dict.""" + # Setup + monkeypatch.setattr("jupyter_matlab_proxy._USE_FALLBACK_KERNEL", False) + monkeypatch.setattr("jupyter_matlab_proxy._MPM_AUTH_TOKEN", "secret") + monkeypatch.setattr("jupyter_matlab_proxy._JUPYTER_SERVER_PID", "123") + mpm_port = 10000 + r = jupyter_matlab_proxy._get_env(mpm_port, None) + assert r.get(mpm_env.get_env_name_mwi_mpm_port()) == str(mpm_port) + assert r.get(mpm_env.get_env_name_mwi_mpm_auth_token()) == "secret" + assert r.get(mpm_env.get_env_name_mwi_mpm_parent_pid()) == "123" + + def test_setup_matlab(): """Tests for a valid Server Process Configuration Dictionary @@ -91,6 +107,41 @@ def test_setup_matlab(): assert os.path.isfile(actual_matlab_setup["launcher_entry"]["icon_path"]) +def test_setup_matlab_with_proxy_manager(monkeypatch): + """Tests for a valid Server Process Configuration Dictionary + + This test checks if the jupyter proxy returns the expected Server Process Configuration + Dictionary for the Matlab process. + """ + + # Setup + monkeypatch.setattr("jupyter_matlab_proxy._USE_FALLBACK_KERNEL", False) + monkeypatch.setattr("jupyter_matlab_proxy._MPM_AUTH_TOKEN", "secret") + monkeypatch.setattr("jupyter_matlab_proxy._JUPYTER_SERVER_PID", "123") + package_path = Path(inspect.getfile(jupyter_matlab_proxy)).parent + icon_path = package_path / "icon_open_matlab.svg" + + expected_matlab_setup = { + "command": [matlab_proxy_manager.get_executable_name()], + "timeout": 100, + "environment": jupyter_matlab_proxy._get_env, + "absolute_url": True, + "launcher_entry": { + "title": "Open MATLAB", + "icon_path": icon_path, + }, + "request_headers_override": { + constants.HEADER_MWI_MPM_CONTEXT: "123", + constants.HEADER_MWI_MPM_AUTH_TOKEN: "secret", + }, + } + + actual_matlab_setup = jupyter_matlab_proxy.setup_matlab() + + assert expected_matlab_setup == actual_matlab_setup + assert os.path.isfile(actual_matlab_setup["launcher_entry"]["icon_path"]) + + def test_setup_matlab_with_token_auth_disabled(monkeypatch): """Tests for a valid Server Process Configuration Dictionary