diff --git a/docs/50_frozen_stubs.md b/docs/50_frozen_stubs.md index f4a7d215..8ca4510c 100644 --- a/docs/50_frozen_stubs.md +++ b/docs/50_frozen_stubs.md @@ -25,9 +25,19 @@ Most OSS firmwares store these frozen modules as part of their repository, which * /GENERIC * /PYBD_SF2 - -3. generate typeshed stubs of these files. (the .pyi files will be stored alongside the .py files) +3. Generate typeshed stubs of these files. + _const pre-processing:_ + As the mypy.stubgen tool is not able to incur the correct types from the MicroPython `foo = const(1)` syntax, + the 'to be frozen' modules are pre-processed usig a regular expression to replace the `foo = const(1)` with `foo = 1`. + If the `.py` files contain any docstrings, they are preserved. Howecer this is uncommon as most micropython-lib modules have not docstrings to save space. + + _Addition of docstrings:_ + Then the docstring to modules, classes and methods are added by merging the docstrings based on the docstubs generated from the MicroPython documentation. + + Finally the stubs are generated using the `stubgen` tool. + The resulting .pyi files are stored alongside the .py files + 4. Include/use them in the configuration diff --git a/mip/full.json b/mip/full.json index e3798f7c..0f62f0a7 100644 --- a/mip/full.json +++ b/mip/full.json @@ -18,5 +18,5 @@ ] ], "deps": [], - "version": "1.17.0" + "version": "1.17.1" } \ No newline at end of file diff --git a/mip/minified.json b/mip/minified.json index 93972d93..b17e236b 100644 --- a/mip/minified.json +++ b/mip/minified.json @@ -18,5 +18,5 @@ ] ], "deps": [], - "version": "1.17.0" + "version": "1.17.1" } \ No newline at end of file diff --git a/mip/mpy_v5.json b/mip/mpy_v5.json index 087b3716..f5dafe14 100644 --- a/mip/mpy_v5.json +++ b/mip/mpy_v5.json @@ -18,5 +18,5 @@ ] ], "deps": [], - "version": "1.17.0" + "version": "1.17.1" } \ No newline at end of file diff --git a/mip/mpy_v6.json b/mip/mpy_v6.json index 578b7277..e8177d07 100644 --- a/mip/mpy_v6.json +++ b/mip/mpy_v6.json @@ -18,5 +18,5 @@ ] ], "deps": [], - "version": "1.17.0" + "version": "1.17.1" } \ No newline at end of file diff --git a/mip/v5/createstubs.py b/mip/v5/createstubs.py index 2ab22f8e..6ce3de0d 100644 --- a/mip/v5/createstubs.py +++ b/mip/v5/createstubs.py @@ -23,7 +23,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/mip/v5/createstubs_db.py b/mip/v5/createstubs_db.py index 38f14d89..e6abbd60 100644 --- a/mip/v5/createstubs_db.py +++ b/mip/v5/createstubs_db.py @@ -18,7 +18,7 @@ - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device -This variant was generated from createstubs.py by micropython-stubber v1.17.0 +This variant was generated from createstubs.py by micropython-stubber v1.17.1 """ # Copyright (c) 2019-2023 Jos Verlinde @@ -42,7 +42,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/mip/v5/createstubs_db_min.py b/mip/v5/createstubs_db_min.py index f3bc2d29..23416079 100644 --- a/mip/v5/createstubs_db_min.py +++ b/mip/v5/createstubs_db_min.py @@ -51,7 +51,7 @@ except O:pass try:from collections import OrderedDict as k except O:from ucollections import OrderedDict as k -__version__='v1.17.0' +__version__='v1.17.1' A3=2 A4=2 A5=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/mip/v5/createstubs_db_mpy.mpy b/mip/v5/createstubs_db_mpy.mpy index 48bd4732..8055f141 100644 Binary files a/mip/v5/createstubs_db_mpy.mpy and b/mip/v5/createstubs_db_mpy.mpy differ diff --git a/mip/v5/createstubs_mem.py b/mip/v5/createstubs_mem.py index 9a434929..e90f75c9 100644 --- a/mip/v5/createstubs_mem.py +++ b/mip/v5/createstubs_mem.py @@ -9,7 +9,7 @@ - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device -This variant was generated from createstubs.py by micropython-stubber v1.17.0 +This variant was generated from createstubs.py by micropython-stubber v1.17.1 """ # Copyright (c) 2019-2023 Jos Verlinde @@ -33,7 +33,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/mip/v5/createstubs_mem_min.py b/mip/v5/createstubs_mem_min.py index 9c2cfd27..aa487c4d 100644 --- a/mip/v5/createstubs_mem_min.py +++ b/mip/v5/createstubs_mem_min.py @@ -47,7 +47,7 @@ except N:pass try:from collections import OrderedDict as f except N:from ucollections import OrderedDict as f -__version__='v1.17.0' +__version__='v1.17.1' y=2 z=2 A0=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/mip/v5/createstubs_mem_mpy.mpy b/mip/v5/createstubs_mem_mpy.mpy index f60d383c..b006a72d 100644 Binary files a/mip/v5/createstubs_mem_mpy.mpy and b/mip/v5/createstubs_mem_mpy.mpy differ diff --git a/mip/v5/createstubs_min.py b/mip/v5/createstubs_min.py index 63ab0568..e0071896 100644 --- a/mip/v5/createstubs_min.py +++ b/mip/v5/createstubs_min.py @@ -50,7 +50,7 @@ except N:pass try:from collections import OrderedDict as g except N:from ucollections import OrderedDict as g -__version__='v1.17.0' +__version__='v1.17.1' A0=2 A1=2 A5=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/mip/v5/createstubs_mpy.mpy b/mip/v5/createstubs_mpy.mpy index 5f0dcebe..05d3eca7 100644 Binary files a/mip/v5/createstubs_mpy.mpy and b/mip/v5/createstubs_mpy.mpy differ diff --git a/mip/v6/createstubs.py b/mip/v6/createstubs.py index 2ab22f8e..6ce3de0d 100644 --- a/mip/v6/createstubs.py +++ b/mip/v6/createstubs.py @@ -23,7 +23,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/mip/v6/createstubs_db.py b/mip/v6/createstubs_db.py index 38f14d89..e6abbd60 100644 --- a/mip/v6/createstubs_db.py +++ b/mip/v6/createstubs_db.py @@ -18,7 +18,7 @@ - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device -This variant was generated from createstubs.py by micropython-stubber v1.17.0 +This variant was generated from createstubs.py by micropython-stubber v1.17.1 """ # Copyright (c) 2019-2023 Jos Verlinde @@ -42,7 +42,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/mip/v6/createstubs_db_min.py b/mip/v6/createstubs_db_min.py index f3bc2d29..23416079 100644 --- a/mip/v6/createstubs_db_min.py +++ b/mip/v6/createstubs_db_min.py @@ -51,7 +51,7 @@ except O:pass try:from collections import OrderedDict as k except O:from ucollections import OrderedDict as k -__version__='v1.17.0' +__version__='v1.17.1' A3=2 A4=2 A5=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/mip/v6/createstubs_db_mpy.mpy b/mip/v6/createstubs_db_mpy.mpy index a1746dc5..d9fc93b1 100644 Binary files a/mip/v6/createstubs_db_mpy.mpy and b/mip/v6/createstubs_db_mpy.mpy differ diff --git a/mip/v6/createstubs_mem.py b/mip/v6/createstubs_mem.py index 9a434929..e90f75c9 100644 --- a/mip/v6/createstubs_mem.py +++ b/mip/v6/createstubs_mem.py @@ -9,7 +9,7 @@ - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device -This variant was generated from createstubs.py by micropython-stubber v1.17.0 +This variant was generated from createstubs.py by micropython-stubber v1.17.1 """ # Copyright (c) 2019-2023 Jos Verlinde @@ -33,7 +33,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/mip/v6/createstubs_mem_min.py b/mip/v6/createstubs_mem_min.py index 9c2cfd27..aa487c4d 100644 --- a/mip/v6/createstubs_mem_min.py +++ b/mip/v6/createstubs_mem_min.py @@ -47,7 +47,7 @@ except N:pass try:from collections import OrderedDict as f except N:from ucollections import OrderedDict as f -__version__='v1.17.0' +__version__='v1.17.1' y=2 z=2 A0=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/mip/v6/createstubs_mem_mpy.mpy b/mip/v6/createstubs_mem_mpy.mpy index 95845954..3d12ccc4 100644 Binary files a/mip/v6/createstubs_mem_mpy.mpy and b/mip/v6/createstubs_mem_mpy.mpy differ diff --git a/mip/v6/createstubs_min.py b/mip/v6/createstubs_min.py index 63ab0568..e0071896 100644 --- a/mip/v6/createstubs_min.py +++ b/mip/v6/createstubs_min.py @@ -50,7 +50,7 @@ except N:pass try:from collections import OrderedDict as g except N:from ucollections import OrderedDict as g -__version__='v1.17.0' +__version__='v1.17.1' A0=2 A1=2 A5=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/mip/v6/createstubs_mpy.mpy b/mip/v6/createstubs_mpy.mpy index 79c022b3..b99fa35a 100644 Binary files a/mip/v6/createstubs_mpy.mpy and b/mip/v6/createstubs_mpy.mpy differ diff --git a/package.json b/package.json index 0559c310..e7cb3519 100644 --- a/package.json +++ b/package.json @@ -18,5 +18,5 @@ ] ], "deps": [], - "version": "1.17.0" + "version": "1.17.1" } diff --git a/pyproject.toml b/pyproject.toml index 6124b824..ad77b454 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ repo-path = "./repos" [tool.poetry] name = "micropython-stubber" -version = "1.17.0" +version = "1.17.1" description = "Tooling to create and maintain stubs for MicroPython" authors = ["Jos Verlinde "] license = "MIT" diff --git a/repos/micropython b/repos/micropython new file mode 160000 index 00000000..0285cb2b --- /dev/null +++ b/repos/micropython @@ -0,0 +1 @@ +Subproject commit 0285cb2bf4a317490364b9249f25ca0eaa6af3ef diff --git a/repos/micropython-lib b/repos/micropython-lib new file mode 160000 index 00000000..7cdf7088 --- /dev/null +++ b/repos/micropython-lib @@ -0,0 +1 @@ +Subproject commit 7cdf70881519c73667efbc4a61a04d9c1a49babb diff --git a/repos/micropython-stubs b/repos/micropython-stubs new file mode 160000 index 00000000..a329b238 --- /dev/null +++ b/repos/micropython-stubs @@ -0,0 +1 @@ +Subproject commit a329b2387f65b4feea0fe9f9a702237370b3cdaa diff --git a/src/stubber/__init__.py b/src/stubber/__init__.py index 7bbc1a30..60d96a31 100644 --- a/src/stubber/__init__.py +++ b/src/stubber/__init__.py @@ -1,4 +1,4 @@ """get the version""" -__version__ = "1.17.0" +__version__ = "1.17.1" diff --git a/src/stubber/board/createstubs.py b/src/stubber/board/createstubs.py index 2ab22f8e..6ce3de0d 100644 --- a/src/stubber/board/createstubs.py +++ b/src/stubber/board/createstubs.py @@ -23,7 +23,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/src/stubber/board/createstubs_db.py b/src/stubber/board/createstubs_db.py index 38f14d89..e6abbd60 100644 --- a/src/stubber/board/createstubs_db.py +++ b/src/stubber/board/createstubs_db.py @@ -18,7 +18,7 @@ - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device -This variant was generated from createstubs.py by micropython-stubber v1.17.0 +This variant was generated from createstubs.py by micropython-stubber v1.17.1 """ # Copyright (c) 2019-2023 Jos Verlinde @@ -42,7 +42,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/src/stubber/board/createstubs_db_min.py b/src/stubber/board/createstubs_db_min.py index f3bc2d29..23416079 100644 --- a/src/stubber/board/createstubs_db_min.py +++ b/src/stubber/board/createstubs_db_min.py @@ -51,7 +51,7 @@ except O:pass try:from collections import OrderedDict as k except O:from ucollections import OrderedDict as k -__version__='v1.17.0' +__version__='v1.17.1' A3=2 A4=2 A5=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/src/stubber/board/createstubs_db_mpy.mpy b/src/stubber/board/createstubs_db_mpy.mpy index 9a532523..79df321c 100644 Binary files a/src/stubber/board/createstubs_db_mpy.mpy and b/src/stubber/board/createstubs_db_mpy.mpy differ diff --git a/src/stubber/board/createstubs_mem.py b/src/stubber/board/createstubs_mem.py index 9a434929..e90f75c9 100644 --- a/src/stubber/board/createstubs_mem.py +++ b/src/stubber/board/createstubs_mem.py @@ -9,7 +9,7 @@ - cross compilation, using mpy-cross, to avoid the compilation step on the micropython device -This variant was generated from createstubs.py by micropython-stubber v1.17.0 +This variant was generated from createstubs.py by micropython-stubber v1.17.1 """ # Copyright (c) 2019-2023 Jos Verlinde @@ -33,7 +33,7 @@ except ImportError: from ucollections import OrderedDict # type: ignore -__version__ = "v1.17.0" +__version__ = "v1.17.1" ENOENT = 2 _MAX_CLASS_LEVEL = 2 # Max class nesting LIBS = ["lib", "/lib", "/sd/lib", "/flash/lib", "."] diff --git a/src/stubber/board/createstubs_mem_min.py b/src/stubber/board/createstubs_mem_min.py index 9c2cfd27..aa487c4d 100644 --- a/src/stubber/board/createstubs_mem_min.py +++ b/src/stubber/board/createstubs_mem_min.py @@ -47,7 +47,7 @@ except N:pass try:from collections import OrderedDict as f except N:from ucollections import OrderedDict as f -__version__='v1.17.0' +__version__='v1.17.1' y=2 z=2 A0=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/src/stubber/board/createstubs_mem_mpy.mpy b/src/stubber/board/createstubs_mem_mpy.mpy index 05ce24a0..6d7f42df 100644 Binary files a/src/stubber/board/createstubs_mem_mpy.mpy and b/src/stubber/board/createstubs_mem_mpy.mpy differ diff --git a/src/stubber/board/createstubs_min.py b/src/stubber/board/createstubs_min.py index 63ab0568..e0071896 100644 --- a/src/stubber/board/createstubs_min.py +++ b/src/stubber/board/createstubs_min.py @@ -50,7 +50,7 @@ except N:pass try:from collections import OrderedDict as g except N:from ucollections import OrderedDict as g -__version__='v1.17.0' +__version__='v1.17.1' A0=2 A1=2 A5=['lib','/lib','/sd/lib','/flash/lib',J] diff --git a/src/stubber/board/createstubs_mpy.mpy b/src/stubber/board/createstubs_mpy.mpy index 49ccf465..424f20cf 100644 Binary files a/src/stubber/board/createstubs_mpy.mpy and b/src/stubber/board/createstubs_mpy.mpy differ diff --git a/src/stubber/commands/get_frozen_cmd.py b/src/stubber/commands/get_frozen_cmd.py index a4237a21..e6cbc031 100644 --- a/src/stubber/commands/get_frozen_cmd.py +++ b/src/stubber/commands/get_frozen_cmd.py @@ -1,4 +1,5 @@ """Get the frozen stubs for MicroPython.""" + ########################################################################################## # get-frozen ########################################################################################## @@ -8,7 +9,6 @@ import click from loguru import logger as log -import stubber.basicgit as git import stubber.utils as utils from stubber.codemod.enrich import enrich_folder from stubber.freeze.get_frozen import freeze_any @@ -76,14 +76,11 @@ def cli_get_frozen( ) ) return -1 - else: - version = utils.clean_version(git.get_local_tag(CONFIG.mpy_path.as_posix()) or "0.0") - if not version: - log.warning("Unable to find the micropython repo in folder : {}".format(CONFIG.mpy_path.as_posix())) - - log.info("MicroPython version : {}".format(version)) - # folder/{family}-{version}-frozen + # folder/{family}-{version[_preview]}-frozen family = "micropython" + # get the current checked out version + version = utils.checkedout_version(CONFIG.mpy_path) + log.info("MicroPython version : {}".format(version)) stub_path = freeze_any(version=version, mpy_path=CONFIG.mpy_path, mpy_lib_path=CONFIG.mpy_lib_path) stub_paths.append(stub_path) diff --git a/src/stubber/freeze/freeze_manifest_2.py b/src/stubber/freeze/freeze_manifest_2.py index 157cdab1..7b45bbac 100644 --- a/src/stubber/freeze/freeze_manifest_2.py +++ b/src/stubber/freeze/freeze_manifest_2.py @@ -10,6 +10,7 @@ from typing import List, Optional from loguru import logger as log + from stubber import utils from stubber.tools.manifestfile import MODE_FREEZE, ManifestFile, ManifestFileError, ManifestOutput from stubber.utils.config import CONFIG @@ -55,10 +56,10 @@ def freeze_one_manifest_2(manifest: Path, frozen_stub_path: Path, mpy_path: Path # save cwd for 'misbehaving' older esp8266 manifest files cwd = Path.cwd() # so we need to get the port and board from the path - log.info(f"input_manifest: {manifest}") + log.debug(f"input_manifest: {manifest}") port, board = get_portboard(manifest) - log.info("port-board: {}".format((port + "-" +board).rstrip("-"))) + log.info("port-board: {}".format((port + "-" + board).rstrip("-"))) path_vars = make_path_vars(port=port, board=board, mpy_path=mpy_path, mpy_lib_path=mpy_lib_path) upy_manifest = ManifestFile(MODE_FREEZE, path_vars) @@ -69,7 +70,7 @@ def freeze_one_manifest_2(manifest: Path, frozen_stub_path: Path, mpy_path: Path except ManifestFileError as er: log.error('freeze error executing "{}": {}'.format(manifest, er.args[0])) raise er - log.info(f"total {len(upy_manifest.files())} files") + log.debug(f"total {len(upy_manifest.files())} files") # restore working directory os.chdir(cwd) @@ -77,7 +78,9 @@ def freeze_one_manifest_2(manifest: Path, frozen_stub_path: Path, mpy_path: Path copy_frozen_to_stubs(frozen_stub_path, port, board, upy_manifest.files(), version, mpy_path=mpy_path) -def copy_frozen_to_stubs(stub_path: Path, port: str, board: str, files: List[ManifestOutput], version: str, mpy_path: Path): +def copy_frozen_to_stubs( + stub_path: Path, port: str, board: str, files: List[ManifestOutput], version: str, mpy_path: Path +): """ copy the frozen files from the manifest to the stubs folder @@ -85,7 +88,7 @@ def copy_frozen_to_stubs(stub_path: Path, port: str, board: str, files: List[Man """ freeze_path, board = get_freeze_path(stub_path, port, board) - log.info(f"copy frozen: {port}-{board} to {freeze_path}") + log.debug(f"copy frozen: {port}-{board} to {freeze_path}") freeze_path.mkdir(parents=True, exist_ok=True) # clean target folder shutil.rmtree(freeze_path, ignore_errors=True) diff --git a/src/stubber/utils/stubmaker.py b/src/stubber/utils/stubmaker.py index c10a593c..3e302b20 100644 --- a/src/stubber/utils/stubmaker.py +++ b/src/stubber/utils/stubmaker.py @@ -1,5 +1,6 @@ """Generate stub files for micropython modules using mypy/stubgen""" +import re import sys from pathlib import Path @@ -63,13 +64,25 @@ def generate_pyi_files(modules_folder: Path) -> bool: """ # stubgen cannot process folders with duplicate modules ( ie v1.14 and v1.15 ) # NOTE: FIX 1 add __init__.py to umqtt - if ( - modules_folder / "umqtt/robust.py" - ).exists(): # and not (freeze_path / "umqtt" / "__init__.py").exists(): + if (modules_folder / "umqtt/robust.py").exists(): log.debug("add missing : umqtt/__init__.py") with open(modules_folder / "umqtt" / "__init__.py", "a") as f: f.write("") + # rx_const = re.compile(r"const\(([\w_\"']+)\)") + rx_const = re.compile(r"const\(([-*<.,:/\(\) \w_\"']+)\)") + # FIX 2 - replace `const(foo)` with `foo` + for f in modules_folder.rglob("*.py"): + if f.is_file(): + with open(f, "r") as file: + data = file.read() + # regex Search for const\(([\w_"']+)\) and replace with (\1) + if rx_const.search(data): + log.debug(f"replace const() in {f}") + data = rx_const.sub(r"\1", data) + with open(f, "w") as file: + file.write(data) + module_list = list(modules_folder.glob("**/modules.json")) r = True if len(module_list) > 1: