get-frozen: preprocess stubs to improve typing for consts

fixes #501

Signed-off-by: Jos Verlinde <jos_verlinde@hotmail.com>

docs/50_frozen_stubs.md

@@ -25,9 +25,19 @@ Most OSS firmwares store these frozen modules as part of their repository, which
        * /GENERIC
        * /PYBD_SF2
 
-       
 
-3. generate typeshed stubs of these files. (the .pyi files will be stored alongside the .py files)
+3. Generate typeshed stubs of these files.
+   _const pre-processing:_
+   As the mypy.stubgen tool is not able to incur the correct types from the MicroPython `foo = const(1)` syntax, 
+   the 'to be frozen' modules are pre-processed usig a regular expression to replace the `foo = const(1)` with `foo = 1`.
+   If the `.py` files contain any docstrings, they are preserved. Howecer this is uncommon as most micropython-lib modules have not docstrings to save space.
+
+    _Addition of docstrings:_
+    Then the docstring to modules, classes and methods are added by merging the docstrings based on the docstubs generated from the MicroPython documentation.
+
+   Finally  the stubs are generated using the `stubgen` tool. 
+   The resulting .pyi files are stored alongside the .py files
+
 
 4. Include/use them in the configuration 
 

src/stubber/commands/get_frozen_cmd.py

@@ -1,4 +1,5 @@
 """Get the frozen stubs for MicroPython."""
+
 ##########################################################################################
 # get-frozen
 ##########################################################################################
@@ -8,7 +9,6 @@
 import click
 from loguru import logger as log
 
-import stubber.basicgit as git
 import stubber.utils as utils
 from stubber.codemod.enrich import enrich_folder
 from stubber.freeze.get_frozen import freeze_any
@@ -76,14 +76,11 @@ def cli_get_frozen(
                 )
             )
             return -1
-    else:
-        version = utils.clean_version(git.get_local_tag(CONFIG.mpy_path.as_posix()) or "0.0")
-    if not version:
-        log.warning("Unable to find the micropython repo in folder : {}".format(CONFIG.mpy_path.as_posix()))
-
-    log.info("MicroPython version : {}".format(version))
-    # folder/{family}-{version}-frozen
+    # folder/{family}-{version[_preview]}-frozen
     family = "micropython"
+    # get the current checked out version
+    version = utils.checkedout_version(CONFIG.mpy_path)
+    log.info("MicroPython version : {}".format(version))
 
     stub_path = freeze_any(version=version, mpy_path=CONFIG.mpy_path, mpy_lib_path=CONFIG.mpy_lib_path)
     stub_paths.append(stub_path)

src/stubber/freeze/freeze_manifest_2.py

@@ -10,6 +10,7 @@
 from typing import List, Optional
 
 from loguru import logger as log
+
 from stubber import utils
 from stubber.tools.manifestfile import MODE_FREEZE, ManifestFile, ManifestFileError, ManifestOutput
 from stubber.utils.config import CONFIG
@@ -55,10 +56,10 @@ def freeze_one_manifest_2(manifest: Path, frozen_stub_path: Path, mpy_path: Path
     # save cwd for 'misbehaving' older esp8266 manifest files
     cwd = Path.cwd()
     # so we need to get the port and board from the path
-    log.info(f"input_manifest: {manifest}")
+    log.debug(f"input_manifest: {manifest}")
     port, board = get_portboard(manifest)
 
-    log.info("port-board: {}".format((port + "-" +board).rstrip("-")))
+    log.info("port-board: {}".format((port + "-" + board).rstrip("-")))
 
     path_vars = make_path_vars(port=port, board=board, mpy_path=mpy_path, mpy_lib_path=mpy_lib_path)
     upy_manifest = ManifestFile(MODE_FREEZE, path_vars)
@@ -69,23 +70,25 @@ def freeze_one_manifest_2(manifest: Path, frozen_stub_path: Path, mpy_path: Path
     except ManifestFileError as er:
         log.error('freeze error executing "{}": {}'.format(manifest, er.args[0]))
         raise er
-    log.info(f"total {len(upy_manifest.files())} files")
+    log.debug(f"total {len(upy_manifest.files())} files")
 
     # restore working directory
     os.chdir(cwd)
     # save the frozen files to the stubs
     copy_frozen_to_stubs(frozen_stub_path, port, board, upy_manifest.files(), version, mpy_path=mpy_path)
 
 
-def copy_frozen_to_stubs(stub_path: Path, port: str, board: str, files: List[ManifestOutput], version: str, mpy_path: Path):
+def copy_frozen_to_stubs(
+    stub_path: Path, port: str, board: str, files: List[ManifestOutput], version: str, mpy_path: Path
+):
     """
     copy the frozen files from the manifest to the stubs folder
 
     stubpath = the destination : # stubs/{family}-{version}-frozen
     """
     freeze_path, board = get_freeze_path(stub_path, port, board)
 
-    log.info(f"copy frozen: {port}-{board} to {freeze_path}")
+    log.debug(f"copy frozen: {port}-{board} to {freeze_path}")
     freeze_path.mkdir(parents=True, exist_ok=True)
     # clean target folder
     shutil.rmtree(freeze_path, ignore_errors=True)

src/stubber/utils/stubmaker.py

@@ -1,5 +1,6 @@
 """Generate stub files for micropython modules using mypy/stubgen"""
 
+import re
 import sys
 from pathlib import Path
 
@@ -63,13 +64,25 @@ def generate_pyi_files(modules_folder: Path) -> bool:
     """
     # stubgen cannot process folders with duplicate modules ( ie v1.14 and v1.15 )
     # NOTE: FIX 1 add __init__.py to umqtt
-    if (
-        modules_folder / "umqtt/robust.py"
-    ).exists():  # and not (freeze_path / "umqtt" / "__init__.py").exists():
+    if (modules_folder / "umqtt/robust.py").exists():
         log.debug("add missing : umqtt/__init__.py")
         with open(modules_folder / "umqtt" / "__init__.py", "a") as f:
             f.write("")
 
+    # rx_const = re.compile(r"const\(([\w_\"']+)\)")
+    rx_const = re.compile(r"const\(([-*<.,:/\(\) \w_\"']+)\)")
+    # FIX 2 - replace `const(foo)` with `foo`
+    for f in modules_folder.rglob("*.py"):
+        if f.is_file():
+            with open(f, "r") as file:
+                data = file.read()
+            # regex Search for const\(([\w_"']+)\) and replace with (\1)
+            if rx_const.search(data):
+                log.debug(f"replace const() in {f}")
+                data = rx_const.sub(r"\1", data)
+                with open(f, "w") as file:
+                    file.write(data)
+
     module_list = list(modules_folder.glob("**/modules.json"))
     r = True
     if len(module_list) > 1: