feat: add compatibility conversion

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.10", "3.11", "3.12", "pypy3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "pypy3.10"]
         exclude:
           - os: windows-latest
             python-version: "3.12"
@@ -20,20 +20,32 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         working-directory: python
         run: |
           python -m pip install -U pip poetry
           poetry install --all-extras --with dev
+
       - name: Run tests
+        if: matrix.python-version != '3.8' && matrix.python-version != '3.9'
         working-directory: python
         run: |
-          poetry run test
+          poetry run test --ignore-glob='*test_compatible.py'
+
+      - name: Run tests (compatible)
+        if: matrix.python-version == '3.8' || matrix.python-version == '3.9'
+        working-directory: python
+        run: |
+          poetry run test "tests/test_compatible.py"
+
       - name: Check Types
+        if: matrix.python-version != '3.8' && matrix.python-version != '3.9'
         working-directory: python
         run: |
           poetry run npx -y pyright

.gitignore

@@ -4,4 +4,11 @@ _*cache*
 .vscode
 *dist*
 *lock*
-**/node_modules
+**/node_modules
+.pytest_cache
+
+# Generated compatible files
+**/promplate/compatible/chain
+**/promplate/compatible/llm
+**/promplate/compatible/prompt
+**/tests/compatible

python/promplate/__init__.py

@@ -1,2 +1,6 @@
-from .chain import *
-from .prompt import *
+try:
+    from .chain import *
+    from .prompt import *
+
+except TypeError:
+    from .compatible import *

python/promplate/compatible/__init__.py

@@ -0,0 +1,90 @@
+from typing import Dict
+import re
+import os
+
+
+def compat_convert(code: str) -> str:
+    """
+    Convert Python 3.10+ code to Python 3.8+ code.
+    """
+
+    # convert `list` type annotation to `List`
+    code = re.sub(r"(:|->) list\[([a-zA-Z0-9_]+) \| ([a-zA-Z0-9_]+)\]", r"\1 List[Union[\2, \3]]", code)
+    code = re.sub(r"(:|->) list\[([a-zA-Z0-9_]+)\]", r"\1 List[\2]", code)
+
+    # convert `|` type annotation to `Union`
+    code = re.sub(r"(:|->) ([a-zA-Z0-9_]+) \| ([a-zA-Z0-9_]+) \| ([a-zA-Z0-9_]+)", r"\1 Union[\2, \3, \4]", code)
+    code = re.sub(r"(:|->) ([a-zA-Z0-9_]+) \| (List\[[a-zA-Z0-9_]+\])  \| ([a-zA-Z0-9_]+)", r"\1 Union[\2, \3, \4]", code)
+    code = re.sub(r"(:|->) ([a-zA-Z0-9_]+) \| ([a-zA-Z0-9_]+)", r"\1 Union[\2, \3]", code)
+    code = re.sub(r"(:|->) (List\[[a-zA-Z0-9_]+\]) \| ([a-zA-Z0-9_]+)", r"\1 Union[\2, \3]", code)
+
+    # convert miscellany
+    code = code.replace("NotRequired", "Optional")
+    code = code.replace("MutableMapping[Any, Any] | None", "Union[MutableMapping[Any, Any], None]")
+    code = code.replace("Mapping[Any, Any] | None", "Union[Mapping[Any, Any], None]")
+    code = code.replace("CTX | None", "Union[CTX, None]")
+    code = code.replace("List[Union[Process, AsyncProcess]] | None", "Union[List[Union[Process, AsyncProcess]], None]")
+    code = code.replace("dict[str, Any]", "Dict[str, Any]")
+
+    # add typing import
+    code = "from typing import *\n" + code
+
+    # replace `promplate` with `promplate.compatible`
+    code = code.replace("from promplate", "from promplate.compatible")
+
+    return code
+
+
+def file_list(root: str) -> Dict[str, str]:
+    """
+    Return a dictionary of all Python files (except this file) under `promplate` directory.
+    This is used to convert all files in `promplate/` to `promplate/compatible/`.
+    """
+
+    assert os.path.basename(root) == "promplate" or os.path.basename(root) == "tests"
+
+    files = {}
+    for path, _, filenames in os.walk(root):
+        if not path.startswith(os.path.join(root, "compatible")):
+            for filename in filenames:
+                if filename.endswith(".py"):
+                    old_path = os.path.join(path, filename)
+                    new_path = os.path.join(
+                        root, "compatible", path[len(root) + 1:], filename
+                    )
+                    files[old_path] = new_path
+
+    # remove `promplate/compatible/__init__.py`
+    if __file__ in files:
+        files.pop(__file__)
+
+    return files
+
+
+def file_list_convert(file_list: Dict[str, str]):
+    """
+    Convert all files in `promplate/` to `promplate/compatible/`.
+    """
+
+    for old_file, new_file in file_list.items():
+        with open(old_file, "r", encoding="utf-8-sig") as f:
+            code = f.read()
+        code = compat_convert(code)
+
+        os.makedirs(os.path.dirname(new_file), exist_ok=True)
+
+        with open(new_file, "w", encoding="utf-8-sig") as f:
+            f.write(code)
+
+
+try:
+    from .chain import *
+    from .prompt import *
+
+except ImportError:
+    # convert all files in `promplate/` to `promplate/compatible/`.
+    root = os.path.dirname(os.path.dirname(__file__))
+    file_list_convert(file_list(root))
+
+    from .chain import *
+    from .prompt import *

python/promplate/prompt/template.py

@@ -52,7 +52,7 @@ def _on_special_token(self, token, sync: bool):
 
         if inner.startswith("end"):
             last = self._ops_stack.pop()
-            assert last == inner.removeprefix("end")
+            assert last == inner[3:]
             self._flush()
             self._builder.dedent()
 

python/pyproject.toml

@@ -16,7 +16,7 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.8"
 typing-extensions = { version = "^4", python = "<3.11" }
 aiohttp = { version = "3.9.0b0", python = ">=3.12", optional = true }
 aiofiles = { version = "^23.2", optional = true }
@@ -38,6 +38,9 @@ pytest = "^7"
 format = "reformat:main"
 test = "pytest:main"
 
+[tool.pyright]
+exclude = ["**/compatible", "tests"]
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

python/tests/test_compatible.py

@@ -0,0 +1,29 @@
+"""Tests for the compatibility of promplate."""
+
+from promplate.compatible.chain.node import Template
+from promplate.compatible import file_list, file_list_convert
+import os
+from subprocess import run
+
+
+def test_compatible():
+    """Test the existence of the `promplate.compatible` module."""
+    template = Template("Hello, {{ name }}!")
+    assert template.render({"name": "World"}) == "Hello, World!"
+
+
+def test_others():
+    """Run all other tests in `tests/`."""
+    test_dir = os.path.dirname(__file__)
+    test_files = file_list(test_dir)
+    test_files.pop(os.path.join(test_dir, "test_compatible.py")) # remove this file
+
+    # convert all files in `tests/` to `tests/compatible/`
+    file_list_convert(test_files)
+
+    # use poetry to run all other tests, and raise error if any test fails
+    for _, test_file in test_files.items():
+        output = run(f"poetry run pytest {test_file}", capture_output=True, shell=True)
+        print(output.stdout.decode())
+        print(output.stderr.decode())
+        assert output.returncode == 0