diff --git a/.coveragerc b/.coveragerc index e7df5c447..bf82390c6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,7 @@ [run] branch = True +omit = + aws_lambda_builders/workflows/python_pip/compat.py [report] exclude_lines = pragma: no cover diff --git a/aws_lambda_builders/__init__.py b/aws_lambda_builders/__init__.py index b8023d8bc..f8a44cd50 100644 --- a/aws_lambda_builders/__init__.py +++ b/aws_lambda_builders/__init__.py @@ -1 +1,4 @@ +""" +AWS Lambda Builder Library +""" __version__ = '0.0.1' diff --git a/aws_lambda_builders/workflow.py b/aws_lambda_builders/workflow.py index 44aa28b27..aeb5948bc 100644 --- a/aws_lambda_builders/workflow.py +++ b/aws_lambda_builders/workflow.py @@ -49,6 +49,7 @@ def __new__(mcs, name, bases, class_dict): if not isinstance(cls.CAPABILITY, Capability): raise ValueError("Workflow '{}' must register valid capabilities".format(cls.NAME)) + LOG.debug("Registering workflow '%s' with capability '%s'", cls.NAME, cls.CAPABILITY) DEFAULT_REGISTRY[cls.CAPABILITY] = cls return cls diff --git a/aws_lambda_builders/workflows/__init__.py b/aws_lambda_builders/workflows/__init__.py index e69de29bb..25258ca3e 100644 --- a/aws_lambda_builders/workflows/__init__.py +++ b/aws_lambda_builders/workflows/__init__.py @@ -0,0 +1,5 @@ +""" +Officially supported builder workflows +""" + +import aws_lambda_builders.workflows.python_pip diff --git a/aws_lambda_builders/workflows/python_pip/__init__.py b/aws_lambda_builders/workflows/python_pip/__init__.py index e69de29bb..13eeb743a 100644 --- a/aws_lambda_builders/workflows/python_pip/__init__.py +++ b/aws_lambda_builders/workflows/python_pip/__init__.py @@ -0,0 +1,5 @@ +""" +Builds Python Lambda functions using PIP dependency manager +""" + +from .workflow import PythonPipWorkflow diff --git a/aws_lambda_builders/workflows/python_pip/actions.py b/aws_lambda_builders/workflows/python_pip/actions.py index 8615da622..0c903b0a0 100644 --- a/aws_lambda_builders/workflows/python_pip/actions.py +++ b/aws_lambda_builders/workflows/python_pip/actions.py @@ -1,10 +1,15 @@ -from aws_lambda_builders.actions import BaseAction -from .packager import PythonPipDependencyBuilder +""" +Action to resolve Python dependencies using PIP +""" + +from aws_lambda_builders.actions import BaseAction, Purpose, ActionFailedError +from .packager import PythonPipDependencyBuilder, PackagerError class PythonPipBuildAction(BaseAction): NAME = 'PythonPipBuildAction' + PURPOSE = Purpose.RESOLVE_DEPENDENCIES def __init__(self, artifacts_dir, manifest_path, runtime): self.artifacts_dir = artifacts_dir @@ -13,8 +18,11 @@ def __init__(self, artifacts_dir, manifest_path, runtime): self.package_builder = PythonPipDependencyBuilder() def execute(self): - self.package_builder.build_dependencies( - self.artifacts_dir, - self.manifest_path, - self.runtime, - ) + try: + self.package_builder.build_dependencies( + self.artifacts_dir, + self.manifest_path, + self.runtime, + ) + except PackagerError as ex: + raise ActionFailedError(str(ex)) diff --git a/aws_lambda_builders/workflows/python_pip/packager.py b/aws_lambda_builders/workflows/python_pip/packager.py index 60ac18d77..37c21afa0 100644 --- a/aws_lambda_builders/workflows/python_pip/packager.py +++ b/aws_lambda_builders/workflows/python_pip/packager.py @@ -1,3 +1,7 @@ +""" +Installs packages using PIP +""" + import sys import re import subprocess @@ -20,24 +24,34 @@ """ -class InvalidSourceDistributionNameError(Exception): +class PackagerError(Exception): + pass + + +class InvalidSourceDistributionNameError(PackagerError): pass -class MissingDependencyError(Exception): +class RequirementsFileNotFoundError(PackagerError): + def __init__(self, requirements_path): + super(RequirementsFileNotFoundError, self).__init__( + 'Requirements file not found: %s' % requirements_path) + + +class MissingDependencyError(PackagerError): """Raised when some dependencies could not be packaged for any reason.""" def __init__(self, missing): self.missing = missing -class NoSuchPackageError(Exception): +class NoSuchPackageError(PackagerError): """Raised when a package name or version could not be found.""" def __init__(self, package_name): super(NoSuchPackageError, self).__init__( 'Could not satisfy the requirement: %s' % package_name) -class PackageDownloadError(Exception): +class PackageDownloadError(PackagerError): """Generic networking error during a package download.""" pass @@ -54,10 +68,12 @@ def __init__(self, osutils=None, dependency_builder=None): :param dependency_builder: This class will be used to build the dependencies of the project. """ + self.osutils = osutils if osutils is None: - osutils = OSUtils() + self.osutils = OSUtils() + if dependency_builder is None: - dependency_builder = DependencyBuilder(osutils) + dependency_builder = DependencyBuilder(self.osutils) self._dependency_builder = dependency_builder def build_dependencies(self, artifacts_dir_path, requirements_path, @@ -93,6 +109,10 @@ def build_dependencies(self, artifacts_dir_path, requirements_path, # correct version of python. We need to enforce that assumption here # by finding/creating a virtualenv of the correct version and when # pip is called set the appropriate env vars. + + if not self.osutils.file_exists(requirements_path): + raise RequirementsFileNotFoundError(requirements_path) + self._dependency_builder.build_site_packages( requirements_path, artifacts_dir_path) diff --git a/aws_lambda_builders/workflows/python_pip/utils.py b/aws_lambda_builders/workflows/python_pip/utils.py index e8db23bd0..3d465e6ad 100644 --- a/aws_lambda_builders/workflows/python_pip/utils.py +++ b/aws_lambda_builders/workflows/python_pip/utils.py @@ -1,3 +1,7 @@ +""" +Commonly used utilities +""" + import io import os import zipfile @@ -10,26 +14,10 @@ class OSUtils(object): - ZIP_DEFLATED = zipfile.ZIP_DEFLATED def environ(self): return os.environ - def open(self, filename, mode): - return open(filename, mode) - - def open_zip(self, filename, mode, compression=ZIP_DEFLATED): - return zipfile.ZipFile(filename, mode, compression=compression) - - def remove_file(self, filename): - """Remove a file, noop if file does not exist.""" - # Unlike os.remove, if the file does not exist, - # then this method does nothing. - try: - os.remove(filename) - except OSError: - pass - def file_exists(self, filename): return os.path.isfile(filename) @@ -46,14 +34,6 @@ def get_file_contents(self, filename, binary=True, encoding='utf-8'): with io.open(filename, mode, encoding=encoding) as f: return f.read() - def set_file_contents(self, filename, contents, binary=True): - if binary: - mode = 'wb' - else: - mode = 'w' - with open(filename, mode) as f: - f.write(contents) - def extract_zipfile(self, zipfile_path, unpack_dir): with zipfile.ZipFile(zipfile_path, 'r') as z: z.extractall(unpack_dir) @@ -71,18 +51,9 @@ def get_directory_contents(self, path): def makedirs(self, path): os.makedirs(path) - def dirname(self, path): - return os.path.dirname(path) - - def abspath(self, path): - return os.path.abspath(path) - def joinpath(self, *args): return os.path.join(*args) - def walk(self, path): - return os.walk(path) - def copytree(self, source, destination): if not os.path.exists(destination): self.makedirs(destination) @@ -98,12 +69,6 @@ def copytree(self, source, destination): def rmtree(self, directory): shutil.rmtree(directory) - def copy(self, source, destination): - shutil.copy(source, destination) - - def move(self, source, destination): - shutil.move(source, destination) - @contextlib.contextmanager def tempdir(self): tempdir = tempfile.mkdtemp() @@ -122,19 +87,3 @@ def mtime(self, path): @property def pipe(self): return subprocess.PIPE - - -class UI(object): - def __init__(self, out=None, err=None): - if out is None: - out = sys.stdout - if err is None: - err = sys.stderr - self._out = out - self._err = err - - def write(self, msg): - self._out.write(msg) - - def error(self, msg): - self._err.write(msg) diff --git a/aws_lambda_builders/workflows/python_pip/workflow.py b/aws_lambda_builders/workflows/python_pip/workflow.py index ca0130210..800d1fabf 100644 --- a/aws_lambda_builders/workflows/python_pip/workflow.py +++ b/aws_lambda_builders/workflows/python_pip/workflow.py @@ -1,4 +1,8 @@ -from aws_lambda_builders.workflow import BaseWorkflow +""" +Python PIP Workflow +""" + +from aws_lambda_builders.workflow import BaseWorkflow, Capability from aws_lambda_builders.actions import CopySourceAction from .actions import PythonPipBuildAction @@ -6,6 +10,11 @@ class PythonPipWorkflow(BaseWorkflow): + NAME = "PythonPipWorkflow" + CAPABILITY = Capability(language="python", + dependency_manager="pip", + application_framework=None) + def __init__(self, source_dir, artifacts_dir, diff --git a/tests/integration/workflows/python_pip/test_python_pip.py b/tests/integration/workflows/python_pip/test_python_pip.py new file mode 100644 index 000000000..c9807db05 --- /dev/null +++ b/tests/integration/workflows/python_pip/test_python_pip.py @@ -0,0 +1,58 @@ + +import os +import shutil +import tempfile +from unittest import TestCase + +from aws_lambda_builders.builder import LambdaBuilder +from aws_lambda_builders.exceptions import WorkflowFailedError + + +class TestPythonPipWorkflow(TestCase): + """ + Verifies that `python_pip` workflow works by building a Lambda that requires Numpy + """ + + TEST_DATA_FOLDER = os.path.join(os.path.dirname(__file__), "testdata") + + def setUp(self): + self.source_dir = self.TEST_DATA_FOLDER + self.artifacts_dir = tempfile.mkdtemp() + self.scratch_dir = tempfile.mkdtemp() + + self.manifest_path_valid = os.path.join(self.TEST_DATA_FOLDER, "requirements-numpy.txt") + self.manifest_path_invalid = os.path.join(self.TEST_DATA_FOLDER, "requirements-invalid.txt") + + self.test_data_files = set(os.listdir(self.TEST_DATA_FOLDER)) + + self.builder = LambdaBuilder(language="python", + dependency_manager="pip", + application_framework=None) + + def tearDown(self): + shutil.rmtree(self.artifacts_dir) + shutil.rmtree(self.scratch_dir) + + def test_must_build_python_project(self): + self.builder.build(self.source_dir, self.artifacts_dir, None, self.manifest_path_valid, + runtime="python2.7") + + expected_files = self.test_data_files.union({"numpy", "numpy-1.15.4.data", "numpy-1.15.4.dist-info"}) + output_files = set(os.listdir(self.artifacts_dir)) + self.assertEquals(expected_files, output_files) + + def test_must_fail_to_resolve_dependencies(self): + + with self.assertRaises(WorkflowFailedError) as ctx: + self.builder.build(self.source_dir, self.artifacts_dir, None, self.manifest_path_invalid, + runtime="python2.7") + + self.assertIn("Invalid requirement: 'adfasf=1.2.3'", str(ctx.exception)) + + def test_must_fail_if_requirements_not_found(self): + + with self.assertRaises(WorkflowFailedError) as ctx: + self.builder.build(self.source_dir, self.artifacts_dir, None, os.path.join("non", "existent", "manifest"), + runtime="python2.7") + + self.assertIn("Requirements file not found", str(ctx.exception)) diff --git a/tests/integration/workflows/python_pip/testdata/__init__.py b/tests/integration/workflows/python_pip/testdata/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/workflows/python_pip/testdata/main.py b/tests/integration/workflows/python_pip/testdata/main.py new file mode 100644 index 000000000..6e295e0fc --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/main.py @@ -0,0 +1,6 @@ +import numpy + + +def lambda_handler(event, context): + # Just return the value of PI with two decimals - 3.14 + return "{0:.2f}".format(numpy.pi) diff --git a/tests/integration/workflows/python_pip/testdata/requirements-invalid.txt b/tests/integration/workflows/python_pip/testdata/requirements-invalid.txt new file mode 100644 index 000000000..3a61c5980 --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/requirements-invalid.txt @@ -0,0 +1 @@ +adfasf=1.2.3 \ No newline at end of file diff --git a/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt new file mode 100644 index 000000000..9b3e78433 --- /dev/null +++ b/tests/integration/workflows/python_pip/testdata/requirements-numpy.txt @@ -0,0 +1 @@ +numpy==1.15.4 diff --git a/tests/unit/workflows/python_pip/test_actions.py b/tests/unit/workflows/python_pip/test_actions.py index e69de29bb..bf2d1e1f8 100644 --- a/tests/unit/workflows/python_pip/test_actions.py +++ b/tests/unit/workflows/python_pip/test_actions.py @@ -0,0 +1,31 @@ + +from unittest import TestCase +from mock import patch + +from aws_lambda_builders.actions import ActionFailedError + +from aws_lambda_builders.workflows.python_pip.actions import PythonPipBuildAction +from aws_lambda_builders.workflows.python_pip.packager import PackagerError + + +class TestPythonPipBuildAction(TestCase): + + @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") + def test_action_must_call_builder(self, PythonPipDependencyBuilderMock): + builder_instance = PythonPipDependencyBuilderMock.return_value + + action = PythonPipBuildAction("artifacts", "manifest", "runtime") + action.execute() + + builder_instance.build_dependencies.assert_called_with("artifacts", "manifest", "runtime") + + @patch("aws_lambda_builders.workflows.python_pip.actions.PythonPipDependencyBuilder") + def test_must_raise_exception_on_failure(self, PythonPipDependencyBuilderMock): + builder_instance = PythonPipDependencyBuilderMock.return_value + builder_instance.build_dependencies.side_effect = PackagerError() + + action = PythonPipBuildAction("artifacts", "manifest", "runtime") + + with self.assertRaises(ActionFailedError): + action.execute() + diff --git a/tests/unit/workflows/python_pip/test_packager.py b/tests/unit/workflows/python_pip/test_packager.py index 098caaff1..0c2b7f21f 100644 --- a/tests/unit/workflows/python_pip/test_packager.py +++ b/tests/unit/workflows/python_pip/test_packager.py @@ -88,8 +88,9 @@ def popen(self, *args, **kwargs): class TestPythonPipDependencyBuilder(object): def test_can_call_dependency_builder(self, osutils): mock_dep_builder = mock.Mock(spec=DependencyBuilder) + osutils_mock = mock.Mock(spec=osutils) builder = PythonPipDependencyBuilder( - osutils=osutils, + osutils=osutils_mock, dependency_builder=mock_dep_builder, ) builder.build_dependencies( @@ -97,6 +98,8 @@ def test_can_call_dependency_builder(self, osutils): ) mock_dep_builder.build_site_packages.assert_called_once_with( 'path/to/requirements.txt', 'artifacts/path/') + osutils_mock.file_exists.assert_called_once_with( + 'path/to/requirements.txt') class TestPackage(object): diff --git a/tests/unit/workflows/python_pip/test_utils.py b/tests/unit/workflows/python_pip/test_utils.py deleted file mode 100644 index 605d88ce2..000000000 --- a/tests/unit/workflows/python_pip/test_utils.py +++ /dev/null @@ -1,20 +0,0 @@ -from six import StringIO - -from aws_lambda_builders.workflows.python_pip import utils - - -class TestUI(object): - def setup(self): - self.out = StringIO() - self.err = StringIO() - self.ui = utils.UI(self.out, self.err) - - def test_write_goes_to_out_obj(self): - self.ui.write("Foo") - assert self.out.getvalue() == 'Foo' - assert self.err.getvalue() == '' - - def test_error_goes_to_err_obj(self): - self.ui.error("Foo") - assert self.err.getvalue() == 'Foo' - assert self.out.getvalue() == ''