From 312af5ab401e53be59713623ffb8287b27bc42bc Mon Sep 17 00:00:00 2001 From: Dustin Spicuzza Date: Sat, 6 Jan 2024 03:22:11 -0500 Subject: [PATCH] Install project requirements at deploy time - Fixes #70 --- robotpy_installer/cli_deploy.py | 191 ++++++++++++++++++++++---------- robotpy_installer/installer.py | 41 +++++-- robotpy_installer/pyproject.py | 59 ++++++++-- 3 files changed, 209 insertions(+), 82 deletions(-) diff --git a/robotpy_installer/cli_deploy.py b/robotpy_installer/cli_deploy.py index 9383cd2..964f4ff 100644 --- a/robotpy_installer/cli_deploy.py +++ b/robotpy_installer/cli_deploy.py @@ -16,7 +16,9 @@ from os.path import join, splitext -from . import sshcontroller +from . import pyproject, sshcontroller +from .installer import PipInstallError, PythonMissingError, RobotpyInstaller +from .errors import Error from .utils import handle_cli_error, print_err, yesno import logging @@ -34,7 +36,10 @@ def wrap_ssh_error(msg: str): class Deploy: """ - Uploads your robot code to the robot and executes it immediately + Installs requirements and uploads code to the robot and executes it immediately + + You must run the 'sync' command first to download the requirements specified + in pyproject.toml. See `robotpy sync --help` for more details. """ def __init__(self, parser: argparse.ArgumentParser): @@ -76,11 +81,26 @@ def __init__(self, parser: argparse.ArgumentParser): ) parser.add_argument( - "-n", - "--no-version-check", + "--ignore-image-version", action="store_true", default=False, - help="If specified, don't verify that your local wpilib install matches the version on the robot (not recommended)", + help="Ignore RoboRIO image version", + ) + + install_args = parser.add_mutually_exclusive_group() + + install_args.add_argument( + "--no-install", + action="store_true", + default=False, + help="If specified, do not use pyproject.toml to install packages on the robot before deploy", + ) + + install_args.add_argument( + "--force-install", + action="store_true", + default=False, + help="Force installation of packages required by pyproject.toml", ) parser.add_argument( @@ -118,7 +138,9 @@ def run( debug: bool, nc: bool, nc_ds: bool, - no_version_check: bool, + ignore_image_version: bool, + no_install: bool, + force_install: bool, large: bool, robot: typing.Optional[str], team: typing.Optional[int], @@ -171,8 +193,14 @@ def run( robot_or_team=robot or team, no_resolve=no_resolve, ) as ssh: - if not self._check_requirements(ssh, no_version_check): - return 1 + self._ensure_requirements( + project_path, + main_file, + ssh, + ignore_image_version, + no_install, + force_install, + ) if not self._do_deploy(ssh, debug, nc, nc_ds, robot_filename, project_path): return 1 @@ -251,67 +279,110 @@ def _check_large_files(self, robot_path: pathlib.Path): return True - def _check_requirements( - self, ssh: sshcontroller.SshController, no_wpilib_version_check: bool - ) -> bool: - # does python exist - with wrap_ssh_error("checking if python exists"): - if ssh.exec_cmd("[ -x /usr/local/bin/python3 ]").returncode != 0: - print_err( - "ERROR: python3 was not found on the roboRIO: have you installed robotpy?" - ) - print_err() - print_err( - f"See {sys.executable} -m robotpy-installer install-python --help" - ) - return False + def _ensure_requirements( + self, + project_path: pathlib.Path, + main_file: pathlib.Path, + ssh: sshcontroller.SshController, + ignore_image_version: bool, + no_install: bool, + force_install: bool, + ): + python_exists = False + requirements_installed = False - # does wpilib exist and does the version match - with wrap_ssh_error("checking for wpilib version"): - py = ";".join( - [ - "import os.path, site", - "version = 'unknown'", - "v = site.getsitepackages()[0] + '/wpilib/version.py'", - "exec(open(v).read(), globals()) if os.path.exists(v) else False", - "print(version)", - ] - ) + project: typing.Optional[pyproject.RobotPyProjectToml] = None - result = ssh.exec_cmd( - f'/usr/local/bin/python3 -c "{py}"', check=True, get_output=True - ) - assert result.stdout is not None + if not no_install: + try: + project = pyproject.load(project_path, default_if_missing=True) + except pyproject.NoRobotpyError as e: + raise pyproject.NoRobotpyError( + f"{e}\n\nUse --no-install to ignore this error (not recommended)" + ) - wpilib_version = result.stdout.strip() - if wpilib_version == "unknown": - print_err( - "WPILib was not found on the roboRIO: have you installed it on the RoboRIO?" + # does python exist + with wrap_ssh_error("checking if python exists"): + python_exists = ( + ssh.exec_cmd("[ -x /usr/local/bin/python3 ]").returncode == 0 + ) + if not python_exists: + logger.warning("Python is not installed on RoboRIO") + + if python_exists: + if no_install: + requirements_installed = True + elif not force_install: + # Use importlib.metadata instead of pip because it's way faster than pip + result = ssh.exec_cmd( + "/usr/local/bin/python3 -c " + "'from importlib.metadata import distributions;" + "import json; import sys; " + "json.dump({dist.name: dist.version for dist in distributions()},sys.stdout)'", + get_output=True, ) - return False + assert result.stdout is not None + pkgdata = json.loads(result.stdout) - print("RoboRIO has WPILib version", wpilib_version) + logger.debug("Roborio has these packages installed:") + for pkg, version in pkgdata.items(): + logger.debug("- %s (%s)", pkg, version) - try: - from wpilib import __version__ as local_wpilib_version # type: ignore - except ImportError: - local_wpilib_version = "unknown" - - if not no_wpilib_version_check and wpilib_version != local_wpilib_version: - print_err(f"ERROR: expected WPILib version {local_wpilib_version}") - print_err() - print_err("You should either:") - print_err( - "- If the robot version is older, upgrade the RobotPy on your robot" + assert project is not None + requirements_installed = pyproject.are_requirements_met( + project, pkgdata ) - print_err("- Otherwise, upgrade pyfrc on your computer") - print_err() - print_err( - "Alternatively, you can specify --no-version-check to skip this check" + if not requirements_installed: + logger.warning("Project requirements not installed on RoboRIO") + else: + logger.info("All project requirements already installed") + + # + # Install requirements + # + + if force_install: + requirements_installed = False + + if not python_exists or not requirements_installed: + if no_install and not python_exists: + raise Error( + "python3 was not found on the roboRIO\n" + "- could not install it because no-install was specified\n" + "- Use 'python -m robotpy installer install-python' to install python separately" ) - return False - return True + installer = RobotpyInstaller() + with installer.connect_to_robot( + project_path=project_path, + main_file=main_file, + ignore_image_version=ignore_image_version, + ssh=ssh, + ): + if not python_exists: + try: + installer.install_python() + except PythonMissingError as e: + raise PythonMissingError( + f"{e}\n\n" + "Run 'python -m robotpy sync' to download your project requirements from the internet (or --no-install to ignore)" + ) from e + + if not requirements_installed: + logger.info("Installing project requirements on RoboRIO:") + assert project is not None + packages = project.get_install_list() + for package in packages: + logger.info("- %s", package) + + try: + installer.pip_install(False, False, False, False, [], packages) + except PipInstallError as e: + raise PipInstallError( + f"{e}\n\n" + "If 'no matching distribution found', run 'python -m robotpy sync' to download your\n" + "project requirements from the internet (or --no-install to ignore)." + ) from e def _do_deploy( self, diff --git a/robotpy_installer/installer.py b/robotpy_installer/installer.py index a7f788d..eb74e09 100755 --- a/robotpy_installer/installer.py +++ b/robotpy_installer/installer.py @@ -45,6 +45,14 @@ class InstallerException(Error): pass +class PipInstallError(InstallerException): + pass + + +class PythonMissingError(InstallerException): + pass + + @contextlib.contextmanager def catch_ssh_error(msg: str): try: @@ -79,15 +87,19 @@ def connect_to_robot( ignore_image_version: bool = False, log_disk_usage: bool = True, no_resolve: bool = False, + ssh: typing.Optional[SshController] = None, ): - ssh = ssh_from_cfg( - project_path, - main_file, - username="admin", - password="", - robot_or_team=robot_or_team, - no_resolve=no_resolve, - ) + if ssh is None: + ssh = ssh_from_cfg( + project_path, + main_file, + username="admin", + password="", + robot_or_team=robot_or_team, + no_resolve=no_resolve, + ) + elif ssh.username != "admin": + ssh = SshController(ssh.hostname, "admin", "") with ssh: self._ssh = ssh @@ -137,7 +149,10 @@ def opkg_install( if package.parent != self.opkg_cache: raise ValueError("internal error") if not package.exists(): - raise InstallerException(f"{package.name} has not been downloaded yet") + raise PythonMissingError( + f"{package.name} has not been downloaded yet\n" + "- Use 'python -m robotpy installer download-python' to download" + ) # Write out the install script # -> we use a script because opkg doesn't have a good mechanism @@ -316,6 +331,9 @@ def _python_ipk_path(self) -> pathlib.Path: parts = urlparse(_PYTHON_IPK) return self.opkg_cache / pathlib.PurePosixPath(parts.path).name + def is_python_downloaded(self) -> bool: + return self._python_ipk_path.exists() + def download_python(self, use_certifi: bool): self.opkg_cache.mkdir(parents=True, exist_ok=True) @@ -328,6 +346,7 @@ def install_python(self): Requires download-python to be executed first. """ + logger.info("Installing Python on RoboRIO (this may take a few minutes)") ipk_dst = self._python_ipk_path self.opkg_install(False, [ipk_dst]) @@ -488,8 +507,10 @@ def pip_install( pip_args.extend(packages) - with catch_ssh_error("installing packages"): + try: self.ssh.exec_cmd(" ".join(pip_args), check=True, print_output=True) + except SshExecError as e: + raise PipInstallError(f"installing packages: {e}") from e # Some of our hacky wheels require this with catch_ssh_error("running ldconfig"): diff --git a/robotpy_installer/pyproject.py b/robotpy_installer/pyproject.py index 757ae8f..ac12cf8 100644 --- a/robotpy_installer/pyproject.py +++ b/robotpy_installer/pyproject.py @@ -5,6 +5,7 @@ import typing from packaging.requirements import Requirement +from packaging.utils import canonicalize_name from packaging.version import Version, InvalidVersion import tomli @@ -15,6 +16,10 @@ class PyprojectError(Error): pass +class NoRobotpyError(PyprojectError): + pass + + def _pyproject_toml_path(project_path: pathlib.Path): return project_path / "pyproject.toml" @@ -56,6 +61,18 @@ def get_install_list(self) -> typing.List[str]: return packages +def robotpy_default_version() -> str: + # this is a bit weird because this project doesn't depend on robotpy, it's + # the other way around.. but oh well? + try: + return metadata("robotpy")["Version"] + except PackageNotFoundError: + raise NoRobotpyError( + "cannot infer default robotpy package version: robotpy package not installed " + "(do `pip install robotpy` or create a pyproject.toml)" + ) from None + + def write_default_pyproject( project_path: pathlib.Path, ): @@ -65,15 +82,7 @@ def write_default_pyproject( :param project_path: Path to robot project """ - # this is a bit weird because this project doesn't depend on robotpy, it's - # the other way around.. but oh well? - try: - robotpy_version = metadata("robotpy")["Version"] - except PackageNotFoundError: - raise PyprojectError( - "cannot infer default robotpy package version: robotpy package not installed " - "(do `pip install robotpy`)" - ) from None + robotpy_version = robotpy_default_version() with open(_pyproject_toml_path(project_path), "w") as fp: fp.write( @@ -104,7 +113,10 @@ def write_default_pyproject( def load( - project_path: pathlib.Path, *, write_if_missing: bool = False + project_path: pathlib.Path, + *, + write_if_missing: bool = False, + default_if_missing=False, ) -> RobotPyProjectToml: """ Reads a pyproject.toml file for a RobotPy project. Raises FileNotFoundError @@ -114,8 +126,13 @@ def load( """ pyproject_path = _pyproject_toml_path(project_path) - if write_if_missing and not pyproject_path.exists(): - write_default_pyproject(project_path) + if not pyproject_path.exists(): + if default_if_missing: + return RobotPyProjectToml( + robotpy_requires=Requirement(f"robotpy=={robotpy_default_version()}") + ) + if write_if_missing: + write_default_pyproject(project_path) with open(pyproject_path, "rb") as fp: data = tomli.load(fp) @@ -166,3 +183,21 @@ def load( requires = [] return RobotPyProjectToml(robotpy_requires=robotpy_requires, requires=requires) + + +def are_requirements_met( + pp: RobotPyProjectToml, packages: typing.Dict[str, str] +) -> bool: + pv = {name: Version(v) for name, v in packages.items()} + for req in [pp.robotpy_requires] + pp.requires: + req_name = canonicalize_name(req.name) + met = False + for pkg, pkg_version in pv.items(): + if pkg == req_name: + met = pkg_version in req.specifier + break + + if not met: + return False + + return True