diff --git a/.appveyor.yml b/.appveyor.yml index 6d70a76fa..8e98574cc 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -18,13 +18,14 @@ install: # To run Nodejs workflow integ tests - ps: Install-Product node 8.10 -- "set PATH=%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%" +- "set PATH=%PYTHON%;%PYTHON%\\Scripts;%PYTHON%\\bin;%PATH%" - "%PYTHON%\\python.exe -m pip install -r requirements/dev.txt" - "%PYTHON%\\python.exe -m pip install -e ." - "set PATH=C:\\Ruby25-x64\\bin;%PATH%" - "gem --version" - "gem install bundler -v 1.17.3 --no-ri --no-rdoc" - "bundler --version" +- "echo %PATH%" # setup go - rmdir c:\go /s /q diff --git a/.pylintrc b/.pylintrc index 2dbf71e74..c30f1c8d6 100644 --- a/.pylintrc +++ b/.pylintrc @@ -9,7 +9,7 @@ # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=compat.py +ignore=compat.py, utils.py # Pickle collected data for later comparisons. persistent=yes @@ -360,4 +360,4 @@ int-import-graph= # Exceptions that will emit a warning when being caught. Defaults to # "Exception" -overgeneral-exceptions=Exception \ No newline at end of file +overgeneral-exceptions=Exception diff --git a/.travis.yml b/.travis.yml index 3eb314b9d..039f25023 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,9 @@ install: # To run Nodejs workflow integ tests - nvm install 8.10.0 - nvm use 8.10.0 + # To run Ruby workflow integ tests + - rvm install ruby-2.5.3 + - rvm use ruby-2.5.3 # Go workflow integ tests require Go 1.11+ - eval "$(gimme 1.11.2)" diff --git a/NOTICE b/NOTICE index 06e1c1af1..674d01f45 100644 --- a/NOTICE +++ b/NOTICE @@ -1,2 +1,6 @@ AWS Lambda Builders Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. + +The function "which" at aws_lambda_builders/utils.py was copied from https://github.com/python/cpython/blob/3.7/Lib/shutil.py +SPDX-License-Identifier: Python-2.0 +Copyright 2019 by the Python Software Foundation \ No newline at end of file diff --git a/aws_lambda_builders/binary_path.py b/aws_lambda_builders/binary_path.py new file mode 100644 index 000000000..78467666f --- /dev/null +++ b/aws_lambda_builders/binary_path.py @@ -0,0 +1,21 @@ +""" +Class containing resolved path of binary given a validator and a resolver and the name of the binary. +""" + + +class BinaryPath(object): + + def __init__(self, resolver, validator, binary, binary_path=None): + self.resolver = resolver + self.validator = validator + self.binary = binary + self._binary_path = binary_path + self.path_provided = True if self._binary_path else False + + @property + def binary_path(self): + return self._binary_path + + @binary_path.setter + def binary_path(self, binary_path): + self._binary_path = binary_path diff --git a/aws_lambda_builders/builder.py b/aws_lambda_builders/builder.py index 1872bfe29..2de4c53ba 100644 --- a/aws_lambda_builders/builder.py +++ b/aws_lambda_builders/builder.py @@ -7,7 +7,6 @@ import logging from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY -from aws_lambda_builders.validate import RuntimeValidator from aws_lambda_builders.workflow import Capability LOG = logging.getLogger(__name__) @@ -91,8 +90,6 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, :param options: Optional dictionary of options ot pass to build action. **Not supported**. """ - if runtime: - self._validate_runtime(runtime) if not os.path.exists(scratch_dir): os.makedirs(scratch_dir) @@ -107,16 +104,5 @@ def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path, return workflow.run() - def _validate_runtime(self, runtime): - """ - validate runtime and local runtime version to make sure they match - - :type runtime: str - :param runtime: - String matching a lambda runtime eg: python3.6 - """ - RuntimeValidator.validate_runtime(required_language=self.capability.language, - required_runtime=runtime) - def _clear_workflows(self): DEFAULT_REGISTRY.clear() diff --git a/aws_lambda_builders/exceptions.py b/aws_lambda_builders/exceptions.py index 656763667..737188510 100644 --- a/aws_lambda_builders/exceptions.py +++ b/aws_lambda_builders/exceptions.py @@ -18,7 +18,7 @@ class UnsupportedManifestError(LambdaBuilderError): class MisMatchRuntimeError(LambdaBuilderError): MESSAGE = "{language} executable found in your path does not " \ "match runtime. " \ - "\n Expected version: {required_runtime}, Found version: {found_runtime}. " \ + "\n Expected version: {required_runtime}, Found version: {runtime_path}. " \ "\n Possibly related: https://github.com/awslabs/aws-lambda-builders/issues/30" diff --git a/aws_lambda_builders/path_resolver.py b/aws_lambda_builders/path_resolver.py new file mode 100644 index 000000000..db7978f2f --- /dev/null +++ b/aws_lambda_builders/path_resolver.py @@ -0,0 +1,28 @@ +""" +Basic Path Resolver that looks for the executable by runtime first, before proceeding to 'language' in PATH. +""" + +from aws_lambda_builders.utils import which + + +class PathResolver(object): + + def __init__(self, binary, runtime): + self.binary = binary + self.runtime = runtime + self.executables = [self.runtime, self.binary] + + def _which(self): + exec_paths = [] + for executable in [executable for executable in self.executables if executable is not None]: + paths = which(executable) + exec_paths.extend(paths) + + if not exec_paths: + raise ValueError("Path resolution for runtime: {} of binary: " + "{} was not successful".format(self.runtime, self.binary)) + return exec_paths + + @property + def exec_paths(self): + return self._which() diff --git a/aws_lambda_builders/utils.py b/aws_lambda_builders/utils.py index e7ebc3941..b33cd8964 100644 --- a/aws_lambda_builders/utils.py +++ b/aws_lambda_builders/utils.py @@ -3,6 +3,7 @@ """ import shutil +import sys import os import logging @@ -57,3 +58,77 @@ def copytree(source, destination, ignore=None): copytree(new_source, new_destination, ignore=ignore) else: shutil.copy2(new_source, new_destination) + +# NOTE: The below function is copied from Python source code and modified +# slightly to return a list of paths that match a given command +# instead of returning just the first match + +# The function "which" at aws_lambda_builders/utils.py was copied from https://github.com/python/cpython/blob/3.7/Lib/shutil.py +# SPDX-License-Identifier: Python-2.0 +# Copyright 2019 by the Python Software Foundation + + +def which(cmd, mode=os.F_OK | os.X_OK, path=None): # pragma: no cover + """Given a command, mode, and a PATH string, return the paths which + conforms to the given mode on the PATH, or None if there is no such + file. + `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result + of os.environ.get("PATH"), or can be overridden with a custom search + path. + Note: This function was backported from the Python 3 source code. + """ + # Check that a given file can be accessed with the correct mode. + # Additionally check that `file` is not a directory, as on Windows + # directories pass the os.access check. + + def _access_check(fn, mode): + return os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn) + + # If we're given a path with a directory part, look it up directly + # rather than referring to PATH directories. This includes checking + # relative to the current directory, e.g. ./script + if os.path.dirname(cmd): + if _access_check(cmd, mode): + return cmd + + return None + + if path is None: + path = os.environ.get("PATH", os.defpath) + if not path: + return None + + path = path.split(os.pathsep) + + if sys.platform == "win32": + # The current directory takes precedence on Windows. + if os.curdir not in path: + path.insert(0, os.curdir) + + # PATHEXT is necessary to check on Windows. + pathext = os.environ.get("PATHEXT", "").split(os.pathsep) + # See if the given file matches any of the expected path + # extensions. This will allow us to short circuit when given + # "python.exe". If it does match, only test that one, otherwise we + # have to try others. + if any(cmd.lower().endswith(ext.lower()) for ext in pathext): + files = [cmd] + else: + files = [cmd + ext for ext in pathext] + else: + # On other platforms you don't have things like PATHEXT to tell you + # what file suffixes are executable, so just pass on cmd as-is. + files = [cmd] + + seen = set() + paths = [] + + for dir in path: + normdir = os.path.normcase(dir) + if normdir not in seen: + seen.add(normdir) + for thefile in files: + name = os.path.join(dir, thefile) + if _access_check(name, mode): + paths.append(name) + return paths diff --git a/aws_lambda_builders/validate.py b/aws_lambda_builders/validate.py deleted file mode 100644 index 82d34d3a9..000000000 --- a/aws_lambda_builders/validate.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Supported Runtimes and their validations. -""" - -import logging -import os -import subprocess - -from aws_lambda_builders.exceptions import MisMatchRuntimeError - -LOG = logging.getLogger(__name__) - - -def validate_python_cmd(required_language, required_runtime_version): - major, minor = required_runtime_version.replace(required_language, "").split('.') - cmd = [ - "python", - "-c", - "import sys; " - "sys.stdout.write('python' + str(sys.version_info.major) + '.' + str(sys.version_info.minor)); " - "assert sys.version_info.major == {major} " - "and sys.version_info.minor == {minor}".format( - major=major, - minor=minor)] - return cmd - - -_RUNTIME_VERSION_RESOLVER = { - "python": validate_python_cmd -} - - -class RuntimeValidator(object): - SUPPORTED_RUNTIMES = [ - "python2.7", - "python3.6", - "python3.7", - ] - - @classmethod - def has_runtime(cls, runtime): - """ - Checks if the runtime is supported. - :param string runtime: Runtime to check - :return bool: True, if the runtime is supported. - """ - return runtime in cls.SUPPORTED_RUNTIMES - - @classmethod - def validate_runtime(cls, required_language, required_runtime): - """ - Checks if the language supplied matches the required lambda runtime - :param string required_language: language to check eg: python - :param string required_runtime: runtime to check eg: python3.6 - :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime - """ - if required_language in _RUNTIME_VERSION_RESOLVER: - if not RuntimeValidator.has_runtime(required_runtime): - LOG.warning("'%s' runtime is not " - "a supported runtime", required_runtime) - return - cmd = _RUNTIME_VERSION_RESOLVER[required_language](required_language, required_runtime) - - p = subprocess.Popen(cmd, - cwd=os.getcwd(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - found_runtime, _ = p.communicate() - if p.returncode != 0: - raise MisMatchRuntimeError(language=required_language, - required_runtime=required_runtime, - found_runtime=str(found_runtime.decode('utf-8'))) - else: - LOG.warning("'%s' runtime has not " - "been validated!", required_language) diff --git a/aws_lambda_builders/validator.py b/aws_lambda_builders/validator.py new file mode 100644 index 000000000..aa0fa1528 --- /dev/null +++ b/aws_lambda_builders/validator.py @@ -0,0 +1,18 @@ +""" +No-op validator that does not validate the runtime_path for a specified language. +""" + +import logging + +LOG = logging.getLogger(__name__) + + +class RuntimeValidator(object): + + def __init__(self, runtime): + self.runtime = runtime + self._runtime_path = None + + def validate(self, runtime_path): + self._runtime_path = runtime_path + return runtime_path diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index 140887de8..fafdcf977 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -1,15 +1,18 @@ """ Implementation of a base workflow """ - +import functools import os import logging from collections import namedtuple import six +from aws_lambda_builders.binary_path import BinaryPath +from aws_lambda_builders.path_resolver import PathResolver +from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.registry import DEFAULT_REGISTRY -from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError +from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError, MisMatchRuntimeError from aws_lambda_builders.actions import ActionFailedError LOG = logging.getLogger(__name__) @@ -22,6 +25,41 @@ Capability = namedtuple('Capability', ["language", "dependency_manager", "application_framework"]) +# TODO: Move sanitize out to its own class. +def sanitize(func): + """ + sanitize the executable path of the runtime specified by validating it. + :param func: Workflow's run method is sanitized + """ + + @functools.wraps(func) + def wrapper(self, *args, **kwargs): + valid_paths = [] + # NOTE: we need to access binaries to get paths and resolvers, before validating. + binaries_copy = self.binaries + for binary, binary_path in binaries_copy.items(): + validator = binary_path.validator + exec_paths = binary_path.resolver.exec_paths if not binary_path.path_provided else binary_path.binary_path + for executable_path in exec_paths: + valid_path = None + try: + valid_path = validator.validate(executable_path) + except MisMatchRuntimeError as ex: + LOG.debug("Invalid executable for %s at %s", + binary, executable_path, exc_info=str(ex)) + if valid_path: + binary_path.binary_path = valid_path + valid_paths.append(valid_path) + break + self.binaries = binaries_copy + if len(self.binaries) != len(valid_paths): + raise WorkflowFailedError(workflow_name=self.NAME, + action_name=None, + reason='Binary validation failed!') + func(self, *args, **kwargs) + return wrapper + + class _WorkflowMetaClass(type): """ A metaclass that maintains the registry of loaded builders @@ -126,6 +164,7 @@ def __init__(self, # Actions are registered by the subclasses as they seem fit self.actions = [] + self._binaries = {} def is_supported(self): """ @@ -138,6 +177,32 @@ def is_supported(self): return True + def get_resolvers(self): + """ + Non specialized path resolver that just returns the list of executable for the runtime on the path. + """ + return [PathResolver(runtime=self.runtime, binary=self.CAPABILITY.language)] + + def get_validators(self): + """ + No-op validator that does not validate the runtime_path. + """ + return [RuntimeValidator(runtime=self.runtime)] + + @property + def binaries(self): + if not self._binaries: + resolvers = self.get_resolvers() + validators = self.get_validators() + self._binaries = {resolver.binary: BinaryPath(resolver=resolver, validator=validator, binary=resolver.binary) + for resolver, validator in zip(resolvers, validators)} + return self._binaries + + @binaries.setter + def binaries(self, binaries): + self._binaries = binaries + + @sanitize def run(self): """ Actually perform the build by executing registered actions. diff --git a/aws_lambda_builders/workflows/nodejs_npm/workflow.py b/aws_lambda_builders/workflows/nodejs_npm/workflow.py index 57396cbc0..dc6be8ea4 100644 --- a/aws_lambda_builders/workflows/nodejs_npm/workflow.py +++ b/aws_lambda_builders/workflows/nodejs_npm/workflow.py @@ -1,7 +1,7 @@ """ NodeJS NPM Workflow """ - +from aws_lambda_builders.path_resolver import PathResolver from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction from .actions import NodejsNpmPackAction, NodejsNpmInstallAction, NodejsNpmrcCopyAction, NodejsNpmrcCleanUpAction @@ -65,3 +65,9 @@ def __init__(self, npm_install, NodejsNpmrcCleanUpAction(artifacts_dir, osutils=osutils) ] + + def get_resolvers(self): + """ + specialized path resolver that just returns the list of executable for the runtime on the path. + """ + return [PathResolver(runtime=self.runtime, binary="npm")] diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index 81d5fe981..e2c0cb3fb 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -3,7 +3,8 @@ """ from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError -from .packager import PythonPipDependencyBuilder, PackagerError +from aws_lambda_builders.workflows.python_pip.utils import OSUtils +from .packager import PythonPipDependencyBuilder, PackagerError, DependencyBuilder, SubprocessPip, PipRunner class PythonPipBuildAction(BaseAction): @@ -11,17 +12,28 @@ class PythonPipBuildAction(BaseAction): NAME = 'ResolveDependencies' DESCRIPTION = "Installing dependencies from PIP" PURPOSE = Purpose.RESOLVE_DEPENDENCIES + LANGUAGE = 'python' - def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime): + def __init__(self, artifacts_dir, manifest_path, scratch_dir, runtime, binaries): self.artifacts_dir = artifacts_dir self.manifest_path = manifest_path self.scratch_dir = scratch_dir self.runtime = runtime - self.package_builder = PythonPipDependencyBuilder(runtime=runtime) + self.binaries = binaries def execute(self): + os_utils = OSUtils() + python_path = self.binaries[self.LANGUAGE].binary_path + pip = SubprocessPip(osutils=os_utils, python_exe=python_path) + pip_runner = PipRunner(python_exe=python_path, pip=pip) + dependency_builder = DependencyBuilder(osutils=os_utils, pip_runner=pip_runner, + runtime=self.runtime) + + package_builder = PythonPipDependencyBuilder(osutils=os_utils, + runtime=self.runtime, + dependency_builder=dependency_builder) try: - self.package_builder.build_dependencies( + package_builder.build_dependencies( self.artifacts_dir, self.manifest_path, self.scratch_dir diff --git a/aws_lambda_builders/workflows/python_pip/compat.py b/aws_lambda_builders/workflows/python_pip/compat.py index 64aba2f06..ff35cbabe 100644 --- a/aws_lambda_builders/workflows/python_pip/compat.py +++ b/aws_lambda_builders/workflows/python_pip/compat.py @@ -1,13 +1,21 @@ import os +from aws_lambda_builders.workflows.python_pip.utils import OSUtils -def pip_import_string(): - import pip - pip_major_version = pip.__version__.split('.')[0] + +def pip_import_string(python_exe): + os_utils = OSUtils() + cmd = [ + python_exe, + "-c", + "import pip; assert int(pip.__version__.split('.')[0]) <= 9" + ] + p = os_utils.popen(cmd,stdout=os_utils.pipe, stderr=os_utils.pipe) + p.communicate() # Pip moved its internals to an _internal module in version 10. # In order to be compatible with version 9 which has it at at the # top level we need to figure out the correct import path here. - if pip_major_version == '9': + if p.returncode == 0: return 'from pip import main' else: return 'from pip._internal import main' diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index a3a2f0130..9ba1dcaa9 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -177,7 +177,7 @@ def __init__(self, osutils, runtime, pip_runner=None): """ self._osutils = osutils if pip_runner is None: - pip_runner = PipRunner(SubprocessPip(osutils)) + pip_runner = PipRunner(python_exe=None, pip=SubprocessPip(osutils)) self._pip = pip_runner self.runtime = runtime @@ -546,12 +546,13 @@ def get_package_name_and_version(self, sdist_path): class SubprocessPip(object): """Wrapper around calling pip through a subprocess.""" - def __init__(self, osutils=None, import_string=None): + def __init__(self, osutils=None, python_exe=None, import_string=None): if osutils is None: osutils = OSUtils() self._osutils = osutils + self.python_exe = python_exe if import_string is None: - import_string = pip_import_string() + import_string = pip_import_string(python_exe=self.python_exe) self._import_string = import_string def main(self, args, env_vars=None, shim=None): @@ -559,12 +560,11 @@ def main(self, args, env_vars=None, shim=None): env_vars = self._osutils.environ() if shim is None: shim = '' - python_exe = sys.executable run_pip = ( 'import sys; %s; sys.exit(main(%s))' ) % (self._import_string, args) exec_string = '%s%s' % (shim, run_pip) - invoke_pip = [python_exe, '-c', exec_string] + invoke_pip = [self.python_exe, '-c', exec_string] p = self._osutils.popen(invoke_pip, stdout=self._osutils.pipe, stderr=self._osutils.pipe, @@ -581,9 +581,10 @@ class PipRunner(object): " Link is a directory," " ignoring download_dir") - def __init__(self, pip, osutils=None): + def __init__(self, python_exe, pip, osutils=None): if osutils is None: osutils = OSUtils() + self.python_exe = python_exe self._wrapped_pip = pip self._osutils = osutils diff --git a/aws_lambda_builders/workflows/python_pip/validator.py b/aws_lambda_builders/workflows/python_pip/validator.py new file mode 100644 index 000000000..e5d44f691 --- /dev/null +++ b/aws_lambda_builders/workflows/python_pip/validator.py @@ -0,0 +1,73 @@ +""" +Python Runtime Validation +""" + +import logging +import os +import subprocess + +from aws_lambda_builders.exceptions import MisMatchRuntimeError + +LOG = logging.getLogger(__name__) + + +class PythonRuntimeValidator(object): + SUPPORTED_RUNTIMES = { + "python2.7", + "python3.6", + "python3.7" + } + + def __init__(self, runtime): + self.language = "python" + self.runtime = runtime + self._valid_runtime_path = None + + def has_runtime(self): + """ + Checks if the runtime is supported. + :param string runtime: Runtime to check + :return bool: True, if the runtime is supported. + """ + return self.runtime in self.SUPPORTED_RUNTIMES + + def validate(self, runtime_path): + """ + Checks if the language supplied matches the required lambda runtime + :param string runtime_path: runtime to check eg: /usr/bin/python3.6 + :raises MisMatchRuntimeError: Version mismatch of the language vs the required runtime + """ + if not self.has_runtime(): + LOG.warning("'%s' runtime is not " + "a supported runtime", self.runtime) + return + + cmd = self._validate_python_cmd(runtime_path) + + p = subprocess.Popen(cmd, + cwd=os.getcwd(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.communicate() + if p.returncode != 0: + raise MisMatchRuntimeError(language=self.language, + required_runtime=self.runtime, + runtime_path=runtime_path) + else: + self._valid_runtime_path = runtime_path + return self._valid_runtime_path + + def _validate_python_cmd(self, runtime_path): + major, minor = self.runtime.replace(self.language, "").split('.') + cmd = [ + runtime_path, + "-c", + "import sys; " + "assert sys.version_info.major == {major} " + "and sys.version_info.minor == {minor}".format( + major=major, + minor=minor)] + return cmd + + @property + def validated_runtime_path(self): + return self._valid_runtime_path if self._valid_runtime_path is not None else None diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index fb2582b0d..a3225ce66 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -1,9 +1,9 @@ """ Python PIP Workflow """ - from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator from .actions import PythonPipBuildAction @@ -65,6 +65,9 @@ def __init__(self, self.actions = [ PythonPipBuildAction(artifacts_dir, scratch_dir, - manifest_path, runtime), + manifest_path, runtime, binaries=self.binaries), CopySourceAction(source_dir, artifacts_dir, excludes=self.EXCLUDED_FILES), ] + + def get_validators(self): + return [PythonRuntimeValidator(runtime=self.runtime)] diff --git a/tests/functional/test_builder.py b/tests/functional/test_builder.py index 553887c0e..29f02415e 100644 --- a/tests/functional/test_builder.py +++ b/tests/functional/test_builder.py @@ -20,7 +20,7 @@ def setUp(self): self.source_dir = tempfile.mkdtemp() self.artifacts_dir = tempfile.mkdtemp() self.scratch_dir = os.path.join(tempfile.mkdtemp(), "scratch") - self.hello_builder = LambdaBuilder(language="test", + self.hello_builder = LambdaBuilder(language="python", dependency_manager="test", application_framework="test", supported_workflows=[ diff --git a/tests/functional/test_cli.py b/tests/functional/test_cli.py index 275ccd411..f044f21d8 100644 --- a/tests/functional/test_cli.py +++ b/tests/functional/test_cli.py @@ -22,7 +22,7 @@ def setUp(self): self.scratch_dir = os.path.join(tempfile.mkdtemp(), "scratch") # Capabilities supported by the Hello workflow - self.language = "test" + self.language = "python" self.dependency_manager = "test" self.application_framework = "test" diff --git a/tests/functional/testdata/workflows/hello_workflow/write_hello.py b/tests/functional/testdata/workflows/hello_workflow/write_hello.py index 5f5bb4c1d..0a8a5e311 100644 --- a/tests/functional/testdata/workflows/hello_workflow/write_hello.py +++ b/tests/functional/testdata/workflows/hello_workflow/write_hello.py @@ -33,7 +33,7 @@ def execute(self): class WriteHelloWorkflow(BaseWorkflow): NAME = "WriteHelloWorkflow" - CAPABILITY = Capability(language="test", dependency_manager="test", application_framework="test") + CAPABILITY = Capability(language="python", dependency_manager="test", application_framework="test") def __init__(self, source_dir, artifacts_dir, *args, **kwargs): super(WriteHelloWorkflow, self).__init__(source_dir, artifacts_dir, *args, **kwargs) diff --git a/tests/functional/workflows/python_pip/test_packager.py b/tests/functional/workflows/python_pip/test_packager.py index 8214ff1be..58618b768 100644 --- a/tests/functional/workflows/python_pip/test_packager.py +++ b/tests/functional/workflows/python_pip/test_packager.py @@ -201,7 +201,9 @@ def environ(self): @pytest.fixture def pip_runner(empty_env_osutils): pip = FakePip() - pip_runner = PipRunner(pip, osutils=empty_env_osutils) + pip_runner = PipRunner(python_exe=sys.executable, + pip=pip, + osutils=empty_env_osutils) return pip, pip_runner @@ -871,7 +873,7 @@ def test_build_into_existing_dir_with_preinstalled_packages( class TestSubprocessPip(object): def test_can_invoke_pip(self): - pip = SubprocessPip() + pip = SubprocessPip(python_exe=sys.executable) rc, out, err = pip.main(['--version']) # Simple assertion that we can execute pip and it gives us some output # and nothing on stderr. @@ -879,7 +881,7 @@ def test_can_invoke_pip(self): assert err == b'' def test_does_error_code_propagate(self): - pip = SubprocessPip() + pip = SubprocessPip(python_exe=sys.executable) rc, _, err = pip.main(['badcommand']) assert rc != 0 # Don't want to depend on a particular error message from pip since it diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py index c4a4f4e41..ed2356d43 100644 --- a/tests/integration/workflows/python_pip/test_python_pip.py +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -6,7 +6,7 @@ from unittest import TestCase from aws_lambda_builders.builder import LambdaBuilder -from aws_lambda_builders.exceptions import WorkflowFailedError, MisMatchRuntimeError +from aws_lambda_builders.exceptions import WorkflowFailedError class TestPythonPipWorkflow(TestCase): @@ -53,13 +53,13 @@ def test_must_build_python_project(self): self.assertEquals(expected_files, output_files) def test_mismatch_runtime_python_project(self): - with self.assertRaises(MisMatchRuntimeError) as mismatch_error: + # NOTE : Build still works if other versions of python are accessible on the path. eg: /usr/bin/python2.7 + # is still accessible within a python 3 virtualenv. + try: self.builder.build(self.source_dir, self.artifacts_dir, self.scratch_dir, self.manifest_path_valid, runtime=self.runtime_mismatch[self.runtime]) - self.assertEquals(mismatch_error.msg, - MisMatchRuntimeError(language="python", - required_runtime=self.runtime_mismatch[self.runtime], - found_runtime=self.runtime).MESSAGE) + except WorkflowFailedError as ex: + self.assertIn("Binary validation failed!", str(ex)) def test_runtime_validate_python_project_fail_open_unsupported_runtime(self): with self.assertRaises(WorkflowFailedError): diff --git a/tests/unit/test_builder.py b/tests/unit/test_builder.py index f611fe1d4..9ef9e8e7e 100644 --- a/tests/unit/test_builder.py +++ b/tests/unit/test_builder.py @@ -115,17 +115,16 @@ def test_with_mocks(self, scratch_dir_exists, get_workflow_mock, importlib_mock, get_workflow_mock.return_value = workflow_cls - with patch.object(LambdaBuilder, "_validate_runtime"): - builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) - - builder.build("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options") - - workflow_cls.assert_called_with("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", - runtime="runtime", optimizations="optimizations", options="options") - workflow_instance.run.assert_called_once() - os_mock.path.exists.assert_called_once_with("scratch_dir") - if scratch_dir_exists: - os_mock.makedirs.not_called() - else: - os_mock.makedirs.assert_called_once_with("scratch_dir") + builder = LambdaBuilder(self.lang, self.lang_framework, self.app_framework, supported_workflows=[]) + + builder.build("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", + runtime="runtime", optimizations="optimizations", options="options") + + workflow_cls.assert_called_with("source_dir", "artifacts_dir", "scratch_dir", "manifest_path", + runtime="runtime", optimizations="optimizations", options="options") + workflow_instance.run.assert_called_once() + os_mock.path.exists.assert_called_once_with("scratch_dir") + if scratch_dir_exists: + os_mock.makedirs.not_called() + else: + os_mock.makedirs.assert_called_once_with("scratch_dir") diff --git a/tests/unit/test_path_resolver.py b/tests/unit/test_path_resolver.py new file mode 100644 index 000000000..3295f6a3c --- /dev/null +++ b/tests/unit/test_path_resolver.py @@ -0,0 +1,27 @@ +from unittest import TestCase + +import os +import mock + +from aws_lambda_builders import utils +from aws_lambda_builders.path_resolver import PathResolver + + +class TestPathResolver(TestCase): + + def setUp(self): + self.path_resolver = PathResolver(runtime="chitti2.0", binary="chitti") + + def test_inits(self): + self.assertEquals(self.path_resolver.runtime, "chitti2.0") + self.assertEquals(self.path_resolver.binary, "chitti") + + def test_which_fails(self): + with self.assertRaises(ValueError): + utils.which = lambda x: None + self.path_resolver._which() + + def test_which_success_immediate(self): + with mock.patch.object(self.path_resolver, '_which') as which_mock: + which_mock.return_value = os.getcwd() + self.assertEquals(self.path_resolver.exec_paths, os.getcwd()) diff --git a/tests/unit/test_validator.py b/tests/unit/test_validator.py new file mode 100644 index 000000000..b3f37d8dd --- /dev/null +++ b/tests/unit/test_validator.py @@ -0,0 +1,15 @@ +from unittest import TestCase + +from aws_lambda_builders.validator import RuntimeValidator + + +class TestRuntimeValidator(TestCase): + + def setUp(self): + self.validator = RuntimeValidator(runtime="chitti2.0") + + def test_inits(self): + self.assertEquals(self.validator.runtime, "chitti2.0") + + def test_validate_runtime(self): + self.validator.validate("/usr/bin/chitti") diff --git a/tests/unit/test_workflow.py b/tests/unit/test_workflow.py index b137f8f13..a6a99cef4 100644 --- a/tests/unit/test_workflow.py +++ b/tests/unit/test_workflow.py @@ -2,6 +2,8 @@ from unittest import TestCase from mock import Mock, call +from aws_lambda_builders.binary_path import BinaryPath +from aws_lambda_builders.validator import RuntimeValidator from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY from aws_lambda_builders.exceptions import WorkflowFailedError, WorkflowUnknownError @@ -151,15 +153,36 @@ def setUp(self): optimizations={"a": "b"}, options={"c": "d"}) + def test_get_binaries(self): + self.assertIsNotNone(self.work.binaries) + for binary, binary_path in self.work.binaries.items(): + self.assertTrue(isinstance(binary_path, BinaryPath)) + + def test_get_validator(self): + self.assertIsNotNone(self.work.get_validators()) + for validator in self.work.get_validators(): + self.assertTrue(isinstance(validator, RuntimeValidator)) + def test_must_execute_actions_in_sequence(self): action_mock = Mock() + validator_mock = Mock() + validator_mock.validate = Mock() + validator_mock.validate.return_value = '/usr/bin/binary' + resolver_mock = Mock() + resolver_mock.exec_paths = ['/usr/bin/binary'] + binaries_mock = Mock() + binaries_mock.return_value = [] + + self.work.get_validators = lambda: validator_mock + self.work.get_resolvers = lambda: resolver_mock self.work.actions = [action_mock.action1, action_mock.action2, action_mock.action3] - + self.work.binaries = {"binary": BinaryPath(resolver=resolver_mock, validator=validator_mock, binary="binary")} self.work.run() self.assertEquals(action_mock.method_calls, [ call.action1.execute(), call.action2.execute(), call.action3.execute() ]) + self.assertTrue(validator_mock.validate.call_count, 1) def test_must_raise_with_no_actions(self): self.work.actions = [] diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index 1f691efff..c2ed21c2b 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -1,8 +1,10 @@ +import sys from unittest import TestCase -from mock import patch +from mock import patch, Mock from aws_lambda_builders.actions import ActionFailedError +from aws_lambda_builders.binary_path import BinaryPath from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction from aws_lambda_builders.workflows.python_pip.packager import PackagerError @@ -15,7 +17,11 @@ def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): builder_instance = PythonPipDependencyBuilderMock.return_value action = PythonPipBuildAction("artifacts", "scratch_dir", - "manifest", "runtime") + "manifest", "runtime", + { + "python": BinaryPath(resolver=Mock(), validator=Mock(), + binary="python", binary_path=sys.executable) + }) action.execute() builder_instance.build_dependencies.assert_called_with("artifacts", @@ -28,7 +34,11 @@ def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): builder_instance.build_dependencies.side_effect = PackagerError() action = PythonPipBuildAction("artifacts", "scratch_dir", - "manifest", "runtime") + "manifest", "runtime", + { + "python": BinaryPath(resolver=Mock(), validator=Mock(), + binary="python", binary_path=sys.executable) + }) with self.assertRaises(ActionFailedError): action.execute() diff --git a/tests/unit/workflows/python_pip/test_builder.py b/tests/unit/workflows/python_pip/test_builder.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 228735e33..e1d58dd74 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -1,3 +1,4 @@ +import sys from collections import namedtuple import mock @@ -47,7 +48,9 @@ def calls(self): def pip_factory(): def create_pip_runner(osutils=None): pip = FakePip() - pip_runner = PipRunner(pip, osutils=osutils) + pip_runner = PipRunner(python_exe=sys.executable, + pip=pip, + osutils=osutils) return pip, pip_runner return create_pip_runner diff --git a/tests/unit/test_runtime.py b/tests/unit/workflows/python_pip/test_validator.py similarity index 50% rename from tests/unit/test_runtime.py rename to tests/unit/workflows/python_pip/test_validator.py index 6f9e01168..6047ab299 100644 --- a/tests/unit/test_runtime.py +++ b/tests/unit/workflows/python_pip/test_validator.py @@ -1,10 +1,10 @@ from unittest import TestCase import mock +from parameterized import parameterized from aws_lambda_builders.exceptions import MisMatchRuntimeError -from aws_lambda_builders.validate import validate_python_cmd -from aws_lambda_builders.validate import RuntimeValidator +from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator class MockSubProcess(object): @@ -13,38 +13,43 @@ def __init__(self, returncode): self.returncode = returncode def communicate(self): - return b'python3,6', None + pass -class TestRuntime(TestCase): +class TestPythonRuntimeValidator(TestCase): - def test_supported_runtimes(self): - self.assertTrue(RuntimeValidator.has_runtime("python2.7")) - self.assertTrue(RuntimeValidator.has_runtime("python3.6")) - self.assertFalse(RuntimeValidator.has_runtime("test_language")) + def setUp(self): + self.validator = PythonRuntimeValidator(runtime='python3.7') - def test_runtime_validate_unsupported_language_fail_open(self): - RuntimeValidator.validate_runtime("test_language", "test_language2.7") + @parameterized.expand([ + "python2.7", + "python3.6", + "python3.7" + ]) + def test_supported_runtimes(self, runtime): + validator = PythonRuntimeValidator(runtime=runtime) + self.assertTrue(validator.has_runtime()) - def test_runtime_validate_unsupported_runtime_version_fail_open(self): - RuntimeValidator.validate_runtime("python", "python2.8") + def test_runtime_validate_unsupported_language_fail_open(self): + validator = PythonRuntimeValidator(runtime='python2.6') + validator.validate(runtime_path='/usr/bin/python2.6') def test_runtime_validate_supported_version_runtime(self): with mock.patch('subprocess.Popen') as mock_subprocess: mock_subprocess.return_value = MockSubProcess(0) - RuntimeValidator.validate_runtime("python", "python3.6") + self.validator.validate(runtime_path='/usr/bin/python3.7') self.assertTrue(mock_subprocess.call_count, 1) def test_runtime_validate_mismatch_version_runtime(self): with mock.patch('subprocess.Popen') as mock_subprocess: mock_subprocess.return_value = MockSubProcess(1) with self.assertRaises(MisMatchRuntimeError): - RuntimeValidator.validate_runtime("python", "python2.7") + self.validator.validate(runtime_path='/usr/bin/python3.6') self.assertTrue(mock_subprocess.call_count, 1) def test_python_command(self): - cmd = validate_python_cmd("python", "python2.7") - version_strings = ["sys.stdout.write", "sys.version_info.major == 2", + cmd = self.validator._validate_python_cmd(runtime_path='/usr/bin/python3.7') + version_strings = ["sys.version_info.major == 3", "sys.version_info.minor == 7"] for version_string in version_strings: - self.assertTrue(any([part for part in cmd if version_string in part])) + self.assertTrue(all([part for part in cmd if version_string in part])) diff --git a/tests/unit/workflows/python_pip/test_workflow.py b/tests/unit/workflows/python_pip/test_workflow.py new file mode 100644 index 000000000..aec99e28b --- /dev/null +++ b/tests/unit/workflows/python_pip/test_workflow.py @@ -0,0 +1,20 @@ +from unittest import TestCase + +from aws_lambda_builders.actions import CopySourceAction +from aws_lambda_builders.workflows.python_pip.validator import PythonRuntimeValidator +from aws_lambda_builders.workflows.python_pip.workflow import PythonPipBuildAction, PythonPipWorkflow + + +class TestPythonPipWorkflow(TestCase): + + def setUp(self): + self.workflow = PythonPipWorkflow("source", "artifacts", "scratch_dir", "manifest", runtime="python3.7") + + def test_workflow_sets_up_actions(self): + self.assertEqual(len(self.workflow.actions), 2) + self.assertIsInstance(self.workflow.actions[0], PythonPipBuildAction) + self.assertIsInstance(self.workflow.actions[1], CopySourceAction) + + def test_workflow_validator(self): + for validator in self.workflow.get_validators(): + self.assertTrue(isinstance(validator, PythonRuntimeValidator))