Merge pull request #162 from Distributive-Network/Xmader/feat/url

`URL`/`URLSearchParams` APIs

pyproject.toml

@@ -29,6 +29,7 @@ include = [
 [tool.poetry.dependencies]
 python = "^3.8"
 pyreadline3 = { version = "^3.4.1", platform = "win32" }
+aiohttp = { version = "^3.8.5", extras = ["speedups"] }
 pminit = { version = "*", allow-prereleases = true }
 
 

python/pminit/pythonmonkey/package.json

@@ -21,6 +21,7 @@
   },
   "homepage": "https://github.com/Distributive-Network/PythonMonkey#readme",
   "dependencies": {
+    "core-js": "^3.32.0",
     "ctx-module": "^1.0.14"
   }
 }

python/pythonmonkey/__init__.py

@@ -9,9 +9,12 @@
 del importlib
 
 # Load the module by default to expose global APIs
+## builtin_modules
 require("console")
 require("base64")
 require("timers")
+require("url")
+require("XMLHttpRequest")
 
 # Add the `.keys()` method on `Object.prototype` to get JSObjectProxy dict() conversion working
 # Conversion from a dict-subclass to a strict dict by `dict(subclass)` internally calls the .keys() method to read the dictionary keys, 

python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @file    XMLHttpRequest-internal.d.ts
+ * @brief   TypeScript type declarations for the internal XMLHttpRequest helpers
+ * @author  Tom Tang <xmader@distributive.network>
+ * @date    August 2023
+ */
+
+/**
+ * `processResponse` callback's argument type
+ */
+export declare interface XHRResponse {
+  /** Response URL */
+  url: string;
+  /** HTTP status */
+  status: number;
+  /** HTTP status message */
+  statusText: string;
+  /** The `Content-Type` header value */
+  contentLength: number;
+  /** Implementation of the `xhr.getResponseHeader` method */
+  getResponseHeader(name: string): string | undefined;
+  /** Implementation of the `xhr.getAllResponseHeaders` method */
+  getAllResponseHeaders(): string;
+  /** Implementation of the `xhr.abort` method */
+  abort(): void;
+}
+
+/**
+ * Send request
+ */
+export declare function request(
+  method: string,
+  url: string,
+  headers: Record<string, string>,
+  body: string | Uint8Array,
+  timeoutMs: number,
+  // callbacks for request body progress
+  processRequestBodyChunkLength: (bytesLength: number) => void,
+  processRequestEndOfBody: () => void,
+  // callbacks for response progress
+  processResponse: (response: XHRResponse) => void,
+  processBodyChunk: (bytes: Uint8Array) => void,
+  processEndOfBody: () => void,
+  // callbacks for known exceptions
+  onTimeoutError: (err: Error) => void,
+  onNetworkError: (err: Error) => void,
+): Promise<void>;
+
+/**
+ * Decode data using the codec registered for encoding.
+ */
+export declare function decodeStr(data: Uint8Array, encoding?: string): string;

python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.py

@@ -0,0 +1,118 @@
+# @file     XMLHttpRequest-internal.py
+# @brief    internal helper functions for XMLHttpRequest
+# @author   Tom Tang <xmader@distributive.network>
+# @date     August 2023
+
+import asyncio
+import aiohttp
+import yarl
+import io
+import platform
+import pythonmonkey as pm
+from typing import Union, ByteString, Callable, TypedDict
+
+class XHRResponse(TypedDict, total=True):
+    """
+    See definitions in `XMLHttpRequest-internal.d.ts`
+    """
+    url: str
+    status: int
+    statusText: str
+    contentLength: int
+    getResponseHeader: Callable[[str], Union[str, None]]
+    getAllResponseHeaders: Callable[[], str]
+    abort: Callable[[], None]
+
+async def request(
+    method: str,
+    url: str,
+    headers: dict,
+    body: Union[str, ByteString],
+    timeoutMs: float,
+    # callbacks for request body progress
+    processRequestBodyChunkLength: Callable[[int], None],
+    processRequestEndOfBody: Callable[[], None],
+    # callbacks for response progress
+    processResponse: Callable[[XHRResponse], None],
+    processBodyChunk: Callable[[bytearray], None],
+    processEndOfBody: Callable[[], None],
+    # callbacks for known exceptions
+    onTimeoutError: Callable[[asyncio.TimeoutError], None],
+    onNetworkError: Callable[[aiohttp.ClientError], None],
+    /
+):
+    class BytesPayloadWithProgress(aiohttp.BytesPayload):
+        _chunkMaxLength = 2**16 # aiohttp default
+
+        async def write(self, writer) -> None:
+            buf = io.BytesIO(self._value)
+            chunk = buf.read(self._chunkMaxLength)
+            while chunk:
+                await writer.write(chunk)
+                processRequestBodyChunkLength(len(chunk))
+                chunk = buf.read(self._chunkMaxLength)
+            processRequestEndOfBody()
+
+    if isinstance(body, str):
+        body = bytes(body, "utf-8")
+
+    # set default headers
+    headers=dict(headers)
+    headers.setdefault("user-agent", f"Python/{platform.python_version()} PythonMonkey/{pm.__version__}")
+
+    if timeoutMs > 0:
+        timeoutOptions = aiohttp.ClientTimeout(total=timeoutMs/1000) # convert to seconds
+    else:
+        timeoutOptions = aiohttp.ClientTimeout() # default timeout
+
+    try:
+        async with aiohttp.request(method=method,
+                                url=yarl.URL(url, encoded=True),
+                                headers=headers,
+                                data=BytesPayloadWithProgress(body) if body else None,
+                                timeout=timeoutOptions,
+        ) as res:
+            def getResponseHeader(name: str):
+                return res.headers.get(name)
+            def getAllResponseHeaders():
+                headers = []
+                for name, value in res.headers.items():
+                    headers.append(f"{name.lower()}: {value}")
+                headers.sort()
+                return "\r\n".join(headers)
+            def abort():
+                res.close()
+
+            # readyState HEADERS_RECEIVED
+            responseData: XHRResponse = { # FIXME: PythonMonkey bug: the dict will be GCed if directly as an argument
+                'url': str(res.real_url),
+                'status': res.status,
+                'statusText': str(res.reason or ''),
+
+                'getResponseHeader': getResponseHeader,
+                'getAllResponseHeaders': getAllResponseHeaders,
+                'abort': abort,
+
+                'contentLength': res.content_length or 0,
+            }
+            processResponse(responseData)
+
+            # readyState LOADING
+            async for data in res.content.iter_any():
+                processBodyChunk(bytearray(data)) # PythonMonkey only accepts the mutable bytearray type
+
+            # readyState DONE
+            processEndOfBody()
+    except asyncio.TimeoutError as e:
+        onTimeoutError(e)
+        raise # rethrow
+    except aiohttp.ClientError as e:
+        onNetworkError(e)
+        raise # rethrow
+
+def decodeStr(data: bytes, encoding='utf-8'): # XXX: Remove this once we get proper TextDecoder support
+    return str(data, encoding=encoding)
+
+# Module exports
+exports['request'] = request # type: ignore
+exports['decodeStr'] = decodeStr # type: ignore