Adding support for file-based reads and externalized type dependencies (

#102)

This is non-breaking change that adds a new public method:

read_files - a file-oriented entry point to the front end. This takes a
list of target DSDL files allowing the user to maintain an explicit list
instead of depending on globular filesystem discovery. Furthermore this
method returns a set which is the transitive closure of types depended
on by the target list of types. This allows consumers to track
dependencies and compiler back ends to generate .d files and otherwise
support incremental builds.

The new method may increase performance for systems with large pools of
messages when generating code for a small sub-set as it only considers
the target and dependencies of the target when parsing dsdl files.

.devcontainer/devcontainer.json

@@ -0,0 +1,24 @@
+{
+    "name": "Python dev environment",
+    "image": "ghcr.io/opencyphal/toxic:tx22.4.2",
+    "workspaceFolder": "/workspace",
+    "workspaceMount": "source=${localWorkspaceFolder},target=/workspace,type=bind,consistency=delegated",
+    "mounts": [
+        "source=root-vscode-server,target=/root/.vscode-server/extensions,type=volume",
+        "source=pydsdl-tox,target=/workspace/.nox,type=volume"
+    ],
+    "customizations": {
+        "vscode": {
+            "extensions": [
+                "uavcan.dsdl",
+                "wholroyd.jinja",
+                "streetsidesoftware.code-spell-checker",
+                "ms-python.python",
+                "ms-python.mypy-type-checker",
+                "ms-python.black-formatter",
+                "ms-python.pylint"
+            ]
+        }
+    },
+    "postCreateCommand": "git submodule update --init --recursive && tox -e local"
+}

.github/workflows/test-and-release.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ ubuntu-latest, macos-latest ]
-        python: [ '3.7', '3.8', '3.9', '3.10', '3.11' ]
+        python: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
         include:
           - os: windows-2019
             python: '3.10'
@@ -60,7 +60,6 @@ jobs:
             echo "${{ runner.os }} not supported"
             exit 1
           fi
-          python -c "import pydsdl; pydsdl.read_namespace('.dsdl-test/uavcan', [])"
         shell: bash
 
   pydsdl-release:

.readthedocs.yml

@@ -2,6 +2,13 @@
 
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.12"
+  apt_packages:
+    - graphviz
+
 sphinx:
   configuration: docs/conf.py
   fail_on_warning: true

conftest.py

@@ -0,0 +1,71 @@
+#
+# Copyright (C) OpenCyphal Development Team  <opencyphal.org>
+# Copyright Amazon.com Inc. or its affiliates.
+# SPDX-License-Identifier: MIT
+#
+"""
+Configuration for pytest tests including fixtures and hooks.
+"""
+
+import tempfile
+from pathlib import Path
+from typing import Any, Optional
+
+import pytest
+
+
+# +-------------------------------------------------------------------------------------------------------------------+
+# | TEST FIXTURES
+# +-------------------------------------------------------------------------------------------------------------------+
+class TemporaryDsdlContext:
+    """
+    Powers the temp_dsdl_factory test fixture.
+    """
+    def __init__(self) -> None:
+        self._base_dir: Optional[Any] = None
+
+    def new_file(self, file_path: Path, text: Optional[str] = None) -> Path:
+        if file_path.is_absolute():
+            raise ValueError(f"{file_path} is an absolute path. The test fixture requires relative paths to work.")
+        file = self.base_dir / file_path
+        file.parent.mkdir(parents=True, exist_ok=True)
+        if text is not None:
+            file.write_text(text)
+        return file
+
+    @property
+    def base_dir(self) -> Path:
+        if self._base_dir is None:
+            self._base_dir = tempfile.TemporaryDirectory()
+        return Path(self._base_dir.name).resolve()
+
+    def _test_path_finalizer(self) -> None:
+        """
+        Finalizer to clean up any temporary directories created during the test.
+        """
+        if self._base_dir is not None:
+            self._base_dir.cleanup()
+            del self._base_dir
+            self._base_dir = None
+
+@pytest.fixture(scope="function")
+def temp_dsdl_factory(request: pytest.FixtureRequest) -> Any:  # pylint: disable=unused-argument
+    """
+    Fixture for pydsdl tests that have to create files as part of the test. This object stays in-scope for a given
+    test method and does not requires a context manager in the test itself.
+
+    Call `new_file(path)` to create a new file path in the fixture's temporary directory. This will create all
+    uncreated parent directories but will _not_ create the file unless text is provided: `new_file(path, "hello")`
+    """
+    f = TemporaryDsdlContext()
+    request.addfinalizer(f._test_path_finalizer)  # pylint: disable=protected-access
+    return f
+
+
+
+@pytest.fixture
+def public_types() -> Path:
+    """
+    Path to the public regulated data types directory used for tests.
+    """
+    return Path(".dsdl-test") / "uavcan"

docs/pages/pydsdl.rst

@@ -12,10 +12,11 @@ You can find a practical usage example in the Nunavut code generation library th
    :local:
 
 
-The main function
-+++++++++++++++++
+The main functions
+++++++++++++++++++
 
 .. autofunction:: pydsdl.read_namespace
+.. autofunction:: pydsdl.read_files
 
 
 Type model

docs/requirements.txt

@@ -1,3 +1,3 @@
-sphinx           == 4.4.0
-sphinx_rtd_theme == 1.0.0
+sphinx           >= 5.0
+sphinx_rtd_theme >= 1.0.0
 sphinx-computron >= 0.2, < 2.0

noxfile.py

@@ -10,7 +10,7 @@
 import nox
 
 
-PYTHONS = ["3.8", "3.9", "3.10", "3.11"]
+PYTHONS = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 """The newest supported Python shall be listed LAST."""
 
 nox.options.error_on_external_run = True
@@ -33,7 +33,6 @@ def clean(session):
         "*.log",
         "*.tmp",
         ".nox",
-        ".dsdl-test",
     ]
     for w in wildcards:
         for f in Path.cwd().glob(w):
@@ -49,9 +48,9 @@ def test(session):
     session.log("Using the newest supported Python: %s", is_latest_python(session))
     session.install("-e", ".")
     session.install(
-        "pytest          ~= 7.3",
-        "pytest-randomly ~= 3.12",
-        "coverage        ~= 7.2",
+        "pytest          ~= 8.1",
+        "pytest-randomly ~= 3.15",
+        "coverage        ~= 7.5",
     )
     session.run("coverage", "run", "-m", "pytest")
     session.run("coverage", "report", "--fail-under=95")
@@ -61,7 +60,7 @@ def test(session):
         session.log(f"OPEN IN WEB BROWSER: file://{report_file}")
 
 
-@nox.session(python=["3.7"])
+@nox.session(python=["3.8"])
 def test_eol(session):
     """This is a minimal test session for those old Pythons that have EOLed."""
     session.install("-e", ".")
@@ -83,30 +82,33 @@ def pristine(session):
 
 @nox.session(python=PYTHONS, reuse_venv=True)
 def lint(session):
-    session.log("Using the newest supported Python: %s", is_latest_python(session))
-    session.install(
-        "mypy   ~= 1.2.0",
-        "pylint ~= 2.17.2",
-    )
-    session.run(
-        "mypy",
-        "--strict",
-        f"--config-file={ROOT_DIR / 'setup.cfg'}",
-        "pydsdl",
-        env={
-            "MYPYPATH": str(THIRD_PARTY_DIR),
-        },
-    )
-    session.run(
-        "pylint",
-        str(ROOT_DIR / "pydsdl"),
-        env={
-            "PYTHONPATH": str(THIRD_PARTY_DIR),
-        },
-    )
+    if is_oldest_python(session):
+        # we run mypy and pylint only on the oldest Python version to ensure maximum compatibility
+        session.install(
+            "mypy   ~= 1.10",
+            "types-parsimonious",
+            "pylint ~= 3.2",
+        )
+        session.run(
+            "mypy",
+            "--strict",
+            f"--config-file={ROOT_DIR / 'setup.cfg'}",
+            "pydsdl",
+            env={
+                "MYPYPATH": str(THIRD_PARTY_DIR),
+            },
+        )
+        session.run(
+            "pylint",
+            str(ROOT_DIR / "pydsdl"),
+            env={
+                "PYTHONPATH": str(THIRD_PARTY_DIR),
+            },
+        )
     if is_latest_python(session):
-        session.install("black ~= 23.3")
-        session.run("black", "--check", ".")
+        # we run black only on the newest Python version to ensure that the code is formatted with the latest version
+        session.install("black ~= 24.4")
+        session.run("black", "--check", f"{ROOT_DIR / 'pydsdl'}")
 
 
 @nox.session(reuse_venv=True)
@@ -121,3 +123,6 @@ def docs(session):
 
 def is_latest_python(session) -> bool:
     return PYTHONS[-1] in session.run("python", "-V", silent=True)
+
+def is_oldest_python(session) -> bool:
+    return PYTHONS[0] in session.run("python", "-V", silent=True)

pydsdl/__init__.py

@@ -7,7 +7,7 @@
 import sys as _sys
 from pathlib import Path as _Path
 
-__version__ = "1.20.1"
+__version__ = "1.21.0"
 __version_info__ = tuple(map(int, __version__.split(".")[:3]))
 __license__ = "MIT"
 __author__ = "OpenCyphal"
@@ -25,8 +25,9 @@
 _sys.path = [str(_Path(__file__).parent / "third_party")] + _sys.path
 
 # Never import anything that is not available here - API stability guarantees are only provided for the exposed items.
+from ._dsdl import PrintOutputHandler as PrintOutputHandler
 from ._namespace import read_namespace as read_namespace
-from ._namespace import PrintOutputHandler as PrintOutputHandler
+from ._namespace import read_files as read_files
 
 # Error model.
 from ._error import FrontendError as FrontendError

pydsdl/_bit_length_set/_symbolic_test.py

@@ -2,7 +2,6 @@
 # This software is distributed under the terms of the MIT License.
 # Author: Pavel Kirienko <pavel@opencyphal.org>
 
-import typing
 import random
 import itertools
 from ._symbolic import NullaryOperator, validate_numerically
@@ -140,7 +139,7 @@ def _unittest_repetition() -> None:
     )
     assert op.min == 7 * 3
     assert op.max == 17 * 3
-    assert set(op.expand()) == set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))  # type: ignore
+    assert set(op.expand()) == set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))
     assert set(op.expand()) == {21, 25, 29, 31, 33, 35, 39, 41, 45, 51}
     assert set(op.modulo(7)) == {0, 1, 2, 3, 4, 5, 6}
     assert set(op.modulo(8)) == {1, 3, 5, 7}
@@ -149,15 +148,15 @@ def _unittest_repetition() -> None:
     for _ in range(1):
         child = NullaryOperator(random.randint(0, 100) for _ in range(random.randint(1, 10)))
         k = random.randint(0, 10)
-        ref = set(map(sum, itertools.combinations_with_replacement(child.expand(), k)))  # type: ignore
+        ref = set(map(sum, itertools.combinations_with_replacement(child.expand(), k)))
         op = RepetitionOperator(child, k)
         assert set(op.expand()) == ref
 
         assert op.min == min(child.expand()) * k
         assert op.max == max(child.expand()) * k
 
         div = random.randint(1, 64)
-        assert set(op.modulo(div)) == {typing.cast(int, x) % div for x in ref}
+        assert set(op.modulo(div)) == {x % div for x in ref}
 
         validate_numerically(op)
 
@@ -173,9 +172,9 @@ def _unittest_range_repetition() -> None:
     assert op.max == 17 * 3
     assert set(op.expand()) == (
         {0}
-        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 1)))  # type: ignore
-        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 2)))  # type: ignore
-        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))  # type: ignore
+        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 1)))
+        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 2)))
+        | set(map(sum, itertools.combinations_with_replacement([7, 11, 17], 3)))
     )
     assert set(op.expand()) == {0, 7, 11, 14, 17, 18, 21, 22, 24, 25, 28, 29, 31, 33, 34, 35, 39, 41, 45, 51}
     assert set(op.modulo(7)) == {0, 1, 2, 3, 4, 5, 6}
@@ -197,10 +196,7 @@ def _unittest_range_repetition() -> None:
         k_max = random.randint(0, 10)
         ref = set(
             itertools.chain(
-                *(
-                    map(sum, itertools.combinations_with_replacement(child.expand(), k))  # type: ignore
-                    for k in range(k_max + 1)
-                )
+                *(map(sum, itertools.combinations_with_replacement(child.expand(), k)) for k in range(k_max + 1))
             )
         )
         op = RangeRepetitionOperator(child, k_max)
@@ -210,7 +206,7 @@ def _unittest_range_repetition() -> None:
         assert op.max == max(child.expand()) * k_max
 
         div = random.randint(1, 64)
-        assert set(op.modulo(div)) == {typing.cast(int, x) % div for x in ref}
+        assert set(op.modulo(div)) == {x % div for x in ref}
 
         validate_numerically(op)