Adding inline script metadata (pep-723) support

setup.cfg

@@ -50,6 +50,7 @@ install_requires =
   click>=6.7,!=7.0
   pip>=9.0.3
   setuptools
+  packaging
 python_requires = >=3.8
 include_package_data = True
 

src/shiv/bootstrap/__init__.py

@@ -18,13 +18,14 @@
 from .interpreter import execute_interpreter
 
 
-def run(module):  # pragma: no cover
+def run(module, inline_script=False):  # pragma: no cover
     """Run a module in a scrubbed environment.
 
     If a single pyz has multiple callers, we want to remove these vars as we no longer need them
     and they can cause subprocesses to fail with a ModuleNotFoundError.
 
     :param Callable module: The entry point to invoke the pyz with.
+    :param bool inline_script: Whether the script is annotated with inline metadata.
     """
     with suppress(KeyError):
         del os.environ[Environment.MODULE]
@@ -35,7 +36,12 @@ def run(module):  # pragma: no cover
     with suppress(KeyError):
         del os.environ[Environment.CONSOLE_SCRIPT]
 
-    sys.exit(module())
+    if inline_script:
+        # inline script will just return a globals dict from the module
+        module()
+        sys.exit()
+    else:
+        sys.exit(module())
 
 
 @contextmanager
@@ -261,6 +267,10 @@ def bootstrap():  # pragma: no cover
         if env.entry_point is not None and not env.script:
             run(import_string(env.entry_point))
 
+        elif env.inline_script is not None:
+            run(partial(runpy.run_path, str(site_packages / "bin" / env.script),
+                        run_name="__main__"), inline_script=True)
+
         elif env.script is not None:
             run(partial(runpy.run_path, str(site_packages / "bin" / env.script), run_name="__main__"))
 

src/shiv/bootstrap/environment.py

@@ -39,6 +39,7 @@ def __init__(
         no_modify: bool = False,
         reproducible: bool = False,
         script: Optional[str] = None,
+        inline_script: Optional[str] = None,
         preamble: Optional[str] = None,
         root: Optional[str] = None,
     ) -> None:
@@ -50,6 +51,7 @@ def __init__(
         self.no_modify: bool = no_modify
         self.reproducible: bool = reproducible
         self.preamble: Optional[str] = preamble
+        self.inline_script: Optional[str] = inline_script
 
         # properties
         self._entry_point: Optional[str] = entry_point

src/shiv/cli.py

@@ -6,13 +6,17 @@
 
 from configparser import ConfigParser
 from datetime import datetime
+from packaging.version import Version
+from packaging.specifiers import SpecifierSet
 from pathlib import Path
+from platform import python_version
 from tempfile import TemporaryDirectory
 from typing import List, Optional
 
 import click
 
 from . import __version__
+from .inline_script import parse_script_metadata
 from . import builder, pip
 from .bootstrap.environment import Environment
 from .constants import (
@@ -22,9 +26,11 @@
     DISALLOWED_PIP_ARGS,
     NO_ENTRY_POINT,
     NO_OUTFILE,
-    NO_PIP_ARGS_OR_SITE_PACKAGES,
+    NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES,
+    SCRIPT_NOT_ANNOTATED,
     SOURCE_DATE_EPOCH_DEFAULT,
     SOURCE_DATE_EPOCH_ENV,
+    MIN_PYTHON_VERSION_ERROR,
 )
 
 
@@ -102,6 +108,12 @@ def copytree(src: Path, dst: Path) -> None:
         "(default is '/usr/bin/env python3')"
     ),
 )
+@click.option(
+    "--inline-script",
+    "-s",
+    help="The path to a PEP-723 inline metadata annotated script to make into the zipapp.",
+    type=click.Path(exists=True),
+)
 @click.option(
     "--site-packages",
     help="The path to an existing site-packages directory to copy into the zipapp.",
@@ -161,6 +173,7 @@ def main(
     entry_point: Optional[str],
     console_script: Optional[str],
     python: Optional[str],
+    inline_script: Optional[str],
     site_packages: Optional[str],
     build_id: Optional[str],
     compressed: bool,
@@ -177,8 +190,8 @@ def main(
     as outlined in PEP 441, but with all their dependencies included!
     """
 
-    if not pip_args and not site_packages:
-        sys.exit(NO_PIP_ARGS_OR_SITE_PACKAGES)
+    if not pip_args and not site_packages and not inline_script:
+        sys.exit(NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES)
 
     if output_file is None:
         sys.exit(NO_OUTFILE)
@@ -213,6 +226,22 @@ def main(
             # Install dependencies into staged site-packages.
             pip.install(["--target", tmp_site_packages] + list(pip_args))
 
+        if inline_script:
+            # Parse the script and add the dependencies to sources
+            metadata = parse_script_metadata(Path(inline_script).read_text())
+            if "script" not in metadata:
+                sys.exit(SCRIPT_NOT_ANNOTATED)
+            script_dependencies = metadata["script"].get("dependencies", [])
+            min_python = metadata["script"].get("requires-python", None)
+            if min_python and Version(python_version()) not in SpecifierSet(min_python):
+                sys.exit(MIN_PYTHON_VERSION_ERROR.format(min_python=min_python, python_version=python_version()))
+            if script_dependencies:
+                pip.install(["--target", tmp_site_packages] + list(script_dependencies))
+            console_script = Path(inline_script).name
+            bin_dir = Path(tmp_site_packages, "bin")
+            bin_dir.mkdir(exist_ok=True)
+            shutil.copy(Path(inline_script).absolute(), bin_dir / console_script)
+
         if preamble:
             bin_dir = Path(tmp_site_packages, "bin")
             bin_dir.mkdir(exist_ok=True)
@@ -252,6 +281,7 @@ def main(
             build_id=build_id,
             entry_point=entry_point,
             script=console_script,
+            inline_script=inline_script,
             compile_pyc=compile_pyc,
             extend_pythonpath=extend_pythonpath,
             shiv_version=__version__,

src/shiv/constants.py

@@ -3,7 +3,9 @@
 
 # errors:
 DISALLOWED_PIP_ARGS = "\nYou supplied a disallowed pip argument! '{arg}'\n\n{reason}\n"
-NO_PIP_ARGS_OR_SITE_PACKAGES = "\nYou must supply PIP ARGS or --site-packages!\n"
+NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES = "\nYou must supply PIP ARGS, --script, or --site-packages!\n"
+SCRIPT_NOT_ANNOTATED = "\nThe provided script is not annotated with PEP-723 metadata!\n"
+MIN_PYTHON_VERSION_ERROR = "\nThe provided script requires Python {min_python}, but you are using {python_version}!\n"
 NO_OUTFILE = "\nYou must provide an output file option! (--output-file/-o)\n"
 NO_ENTRY_POINT = "\nNo entry point '{entry_point}' found in console_scripts or the bin dir!\n"
 BINPRM_ERROR = "\nShebang is too long, it would exceed BINPRM_BUF_SIZE! Consider /usr/bin/env\n"

src/shiv/inline_script.py

@@ -0,0 +1,31 @@
+import re
+try:
+    # python 3.11+ has tomllib in stdlib
+    import tomllib  # type: ignore
+except (ModuleNotFoundError, ImportError):
+    # python 3.8-3.10, use pip vendored tomli
+    import pip._vendor.tomli as tomllib  # type: ignore
+
+REGEX = r'(?m)^# /// (?P<type>[a-zA-Z0-9-]+)$\s(?P<content>(^#(| .*)$\s)+)^# ///$'
+
+
+def parse_script_metadata(script: str) -> dict:
+    """Parses the metadata from a PEP-723 annotated script.
+
+    The metadata is stored in a nested dictionary structure.
+    The only PEP defined metadata type is "script", which contains the
+    dependencies of the script and minimum Python version.
+
+    :param script: The text of the script to parse.
+    """
+
+    metadata = {}
+
+    for match in re.finditer(REGEX, script):
+        md_type, content = match.group('type'), ''.join(
+            line[2:] if line.startswith('# ') else line[1:]
+            for line in match.group('content').splitlines(keepends=True)
+        )
+        metadata[md_type] = tomllib.loads(content)
+
+    return metadata

test/script/deps.py

@@ -0,0 +1,17 @@
+# /// script
+# dependencies = [
+#   "pyyaml<7",
+#   "rich",
+# ]
+# ///
+
+import yaml
+from rich.pretty import pprint
+
+document = """
+  hello: world
+  foo:
+    bar: 1
+    baz: 2
+"""
+pprint(yaml.safe_load(document))

test/script/deps_and_python.py

@@ -0,0 +1,29 @@
+# /// script
+# requires-python = ">=3.8"
+# dependencies = [
+#   "python-dateutil",
+#   "rich",
+# ]
+# ///
+
+# Code from https://pypi.org/project/python-dateutil/
+# Description:
+# Suppose you want to know how much time is left, in years/months/days/etc,
+# before the next easter happening on a year with a Friday 13th in August,
+# and you want to get today’s date out of the “date” unix system command.
+
+from dateutil.relativedelta import relativedelta, FR
+from dateutil.easter import easter
+from dateutil.rrule import rrule, YEARLY
+from dateutil.parser import parse
+from rich.pretty import pprint
+
+now = parse("Sat Oct 11 17:13:46 UTC 2003")
+today = now.date()
+year = rrule(YEARLY, dtstart=now, bymonth=8, bymonthday=13, byweekday=FR)[0].year
+rdelta = relativedelta(easter(year), today)
+
+pprint(f"Today is: {today}")
+pprint(f"Year with next Aug 13th on a Friday is: {year}")
+pprint(f"How far is the Easter of that year: {rdelta}")
+pprint(f"And the Easter of that year is: {today+rdelta}")

test/script/min_python.py

@@ -0,0 +1,17 @@
+# /// script
+# requires-python = ">=3.8"
+# ///
+
+# No external dependencies
+
+import json
+
+document = {
+    "hello": "world",
+    "foo": {
+        "bar": 1,
+        "baz": 2,
+    },
+}
+
+print(json.dumps(document))

test/test_cli.py

@@ -12,7 +12,7 @@
 
 from click.testing import CliRunner
 from shiv.cli import console_script_exists, find_entry_point, main
-from shiv.constants import DISALLOWED_ARGS, DISALLOWED_PIP_ARGS, NO_OUTFILE, NO_PIP_ARGS_OR_SITE_PACKAGES
+from shiv.constants import DISALLOWED_ARGS, DISALLOWED_PIP_ARGS, NO_OUTFILE, NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES
 from shiv.info import main as info_main
 from shiv.pip import install
 
@@ -76,7 +76,7 @@ def test_no_args(self, runner):
         result = runner([])
 
         assert result.exit_code == 1
-        assert NO_PIP_ARGS_OR_SITE_PACKAGES in result.output
+        assert NO_PIP_ARGS_SCRIPT_OR_SITE_PACKAGES in result.output
 
     def test_no_outfile(self, runner):
         """This should fail with a warning about not providing an outfile"""
@@ -384,3 +384,31 @@ def test_alternate_root_environment_variable(self, runner, package_location, tmp
         assert proc.returncode == 0
         assert "hello" in proc.stdout.decode()
         assert shiv_root_path.exists()
+
+    @pytest.mark.parametrize(
+        "script_location, expected_output",
+        [
+            ("test/script/deps_and_python.py", ["2003-10-11", "2004-04-11"]),
+            ("test/script/min_python.py", ["hello", "world"]),
+            ("test/script/deps.py", ["foo", "bar"]),
+        ],
+    )
+    def test_inline_script(self, script_location, expected_output, runner, tmp_path):
+        """Test that the --inline-script argument works."""
+
+        output_file = tmp_path / "test.pyz"
+        result = runner(["--inline-script", script_location, "-o", str(output_file)])
+
+        # check that the command successfully completed
+        assert result.exit_code == 0
+
+        # ensure the created file actually exists
+        assert output_file.exists()
+
+        # now run the produced zipapp
+        proc = subprocess.run(
+            [str(output_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ,
+        )
+
+        assert proc.returncode == 0
+        assert all(expected in proc.stdout.decode() for expected in expected_output)

test/test_inline_script.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+import pytest
+
+from shiv.inline_script import parse_script_metadata
+
+
+class TestInlineScript:
+    @pytest.mark.parametrize(
+        "script_location,expected_metadata",
+        [
+            ("test/script/deps.py", {
+                "script": {
+                    "dependencies": [
+                        "pyyaml<7",
+                        "rich"
+                    ],
+                }
+            }),
+            ("test/script/min_python.py", {
+                "script": {
+                    "requires-python": ">=3.8",
+                }
+            }),
+            ("test/script/deps_and_python.py", {
+                "script": {
+                    "requires-python": ">=3.8",
+                    "dependencies": [
+                        "python-dateutil",
+                        "rich",
+                    ],
+                }
+            }),
+            ("test/package/hello/__init__.py", {}),
+        ],
+    )
+    def test_parse_script_metadata(self, script_location, expected_metadata):
+        script_text = Path(script_location).read_text()
+        assert parse_script_metadata(script_text) == expected_metadata

tox.ini

@@ -1,13 +1,15 @@
 [tox]
 isolated_build = True
-envlist = py38, py39, py310, py311
+envlist = py38, py39, py310, py311, py312
+
 
 [gh-actions]
 python =
   3.8: py38
   3.9: py39
   3.10: py310
   3.11: py311
+  3.12: py312
 
 [testenv]
 commands=