Builder interface with ability to dynamically load actions & run (#6)

DESIGN.md

@@ -63,6 +63,58 @@ customers with a standard expectation.
 This library provides a wrapper CLI interface for convenience. This interface is not supported at the moment. So we 
 don't provide any guarantees of back compatibility. 
 
+It is a very thin wrapper over the library. It is meant to integrate
+with tools written in other programming languages that can't import Python libraries directly. The CLI provides
+a JSON-RPC interface over stdin/stdout to invoke the builder and get response.
+
+**Request Format**
+
+```json
+{
+  "jsonrpc": "2.0",
+  "method": "LambdaBuilder.build",
+  "id": 1,
+  "params": {
+    "capability": {
+      "language": "<programming language>",
+      "dependency_manager": "<programming language framework>",
+      "application_framework": "<application framework>"
+    },
+    "source_dir": "/path/to/source",
+    "artifacts_dir": "/path/to/store/artifacts",
+    "scratch_dir": "/path/to/tmp",
+    "manifest_path": "/path/to/manifest.json",
+    "runtime": "Function's runtime ex: nodejs8.10",
+    "optimizations": {},  // not supported 
+    "options": {}  // not supported
+  }
+}
+```
+
+**Successful Response Format**
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "result": {}  // Empty result indicates success
+}
+```
+
+**Error Response Format**
+
+```json
+{
+  "jsonrpc": "2.0",
+  "id": 1,
+  "error": {
+    "code": "",  // Integer code indicating the problem
+    "message": "",  // Contains the Exception name 
+    "data": ""   // Contains the exception message
+  }  
+}
+```
+
 ### Project Meta
 #### Directory Structure
 This project's directories are laid as follows:

Makefile

@@ -4,7 +4,10 @@ init:
 test:
 	# Run unit tests
 	# Fail if coverage falls below 95%
-	pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 95 tests/unit tests/functional
+	LAMBDA_BUILDERS_DEV=1 pytest --cov aws_lambda_builders --cov-report term-missing --cov-fail-under 95 tests/unit tests/functional
+
+func-test:
+	LAMBDA_BUILDERS_DEV=1 pytest tests/functional
 
 integ-test:
 	# Integration tests don't need code coverage

aws_lambda_builders/__main__.py

@@ -0,0 +1,65 @@
+"""
+CLI interface for AWS Lambda Builder. It is a very thin wrapper over the library. It is meant to integrate
+with tools written in other programming languages that can't import Python libraries directly. The CLI provides
+a JSON-RPC interface over stdin/stdout to invoke the builder and get response.
+
+Read the design document for explanation of the JSON-RPC interface
+"""
+
+import sys
+import json
+from aws_lambda_builders.builder import LambdaBuilder
+from aws_lambda_builders.exceptions import WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError
+
+
+def _success_response(request_id):
+    return json.dumps({
+        "jsonrpc": "2.0",
+        "id": request_id,
+        "result": {}
+    })
+
+
+def main():
+    """
+    Implementation of CLI Interface. Handles only one JSON-RPC method at a time and responds with data
+    """
+
+    # For now the request is not validated
+    request = json.load(sys.stdin)
+
+    request_id = request["id"]
+    params = request["params"]
+    capabilities = params["capability"]
+    supported_workflows = params["supported_workflows"]
+
+    try:
+        builder = LambdaBuilder(language=capabilities["language"],
+                                dependency_manager=capabilities["dependency_manager"],
+                                application_framework=capabilities["application_framework"],
+                                supported_workflows=supported_workflows)
+
+        builder.build(params["source_dir"],
+                      params["artifacts_dir"],
+                      params["scratch_dir"],
+                      params["manifest_path"],
+                      runtime=params["runtime"],
+                      optimizations=params["optimizations"],
+                      options=params["options"])
+
+        # Return a success response
+        sys.stdout.write(_success_response(request_id))
+        sys.stdout.flush()  # Make sure it is written
+
+    except (WorkflowNotFoundError, WorkflowUnknownError, WorkflowFailedError) as ex:
+        # TODO: Return a workflow error response
+        print(str(ex))
+        sys.exit(1)
+    except Exception as ex:
+        # TODO: Return a internal server response
+        print(str(ex))
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

aws_lambda_builders/actions.py

@@ -2,28 +2,100 @@
 Definition of actions used in the workflow
 """
 
-class BaseAction(object):
+import logging
+import shutil
+import six
+
+from aws_lambda_builders.utils import copytree
+
+LOG = logging.getLogger(__name__)
+
+
+class ActionFailedError(Exception):
+    """
+    Base class for exception raised when action failed to complete. Use this to express well-known failure scenarios.
+    """
+    pass
+
+
+class Purpose(object):
+    """
+    Enum like object to describe the purpose of each action.
+    """
+
+    # Action is identifying dependencies, downloading, compiling and resolving them
+    RESOLVE_DEPENDENCIES = "RESOLVE_DEPENDENCIES"
+
+    # Action is copying source code
+    COPY_SOURCE = "COPY_SOURCE"
+
+    # Action is compiling source code
+    COMPILE_SOURCE = "COMPILE_SOURCE"
+
+    @staticmethod
+    def has_value(item):
+        return item in Purpose.__dict__.values()
+
+
+class _ActionMetaClass(type):
+
+    def __new__(mcs, name, bases, class_dict):
+
+        cls = type.__new__(mcs, name, bases, class_dict)
+
+        if cls.__name__ == 'BaseAction':
+            return cls
+
+        # Validate class variables
+        # All classes must provide a name
+        if not isinstance(cls.NAME, six.string_types):
+            raise ValueError("Action must provide a valid name")
+
+        if not Purpose.has_value(cls.PURPOSE):
+            raise ValueError("Action must provide a valid purpose")
+
+        return cls
+
+
+class BaseAction(six.with_metaclass(_ActionMetaClass, object)):
     """
     Base class for all actions. It does not provide any implementation.
     """
 
-    NAME = 'BaseAction'
+    # Every action must provide a name
+    NAME = None
+
+    # Optional description explaining what this action is about. Used to print help text
+    DESCRIPTION = ""
+
+    # What is this action meant for? Must be a valid instance of `Purpose` class
+    PURPOSE = None
 
     def execute(self):
         """
-        The builder will run this method on each action. This method should complete the action, and if it fails
-        raise appropriate exceptions.
+        Runs the action. This method should complete the action, and if it fails raise appropriate exceptions.
 
-        :raises lambda_builders.exceptions.ActionError: Instance of this class if something went wrong with the
+        :raises lambda_builders.actions.ActionFailedError: Instance of this class if something went wrong with the
             action
         """
-        raise NotImplementedError()
+        raise NotImplementedError('execute')
+
+    def __repr__(self):
+        return "Name={}, Purpose={}, Description={}".format(self.NAME, self.PURPOSE, self.DESCRIPTION)
 
 
 class CopySourceAction(BaseAction):
 
+    NAME = 'CopySourceAction'
+
+    DESCRIPTION = "Copies source code while skipping certain commonly excluded files"
+
+    PURPOSE = Purpose.COPY_SOURCE
+
     def __init__(self, source_dir, dest_dir, excludes=None):
-        pass
+        self.source_dir = source_dir
+        self.dest_dir = dest_dir
+        self.excludes = excludes or []
 
     def execute(self):
-        pass
+        copytree(self.source_dir, self.dest_dir, ignore=shutil.ignore_patterns(*self.excludes))

aws_lambda_builders/builder.py

@@ -0,0 +1,104 @@
+"""
+Entrypoint for the AWS Lambda Builder library
+"""
+
+import importlib
+import logging
+
+from aws_lambda_builders.registry import get_workflow, DEFAULT_REGISTRY
+from aws_lambda_builders.workflow import Capability
+
+LOG = logging.getLogger(__name__)
+
+_SUPPORTED_WORKFLOWS = [
+    "aws_lambda_builders.workflows"
+]
+
+
+class LambdaBuilder(object):
+    """
+    Helps you build AWS Lambda functions. This class is the primary entry point for this library.
+    """
+
+    def __init__(self, language, dependency_manager, application_framework, supported_workflows=None):
+
+        """
+        Initialize the builder.
+        :type supported_workflows: list
+        :param supported_workflows:
+            Optional list of workflow modules that should be loaded. By default we load all the workflows bundled
+            with this library. This property is primarily used for testing. But in future it could be used to
+            dynamically load user defined workflows.
+
+            If set to None, we will load the default workflow modules.
+            If set to empty list, we will **not** load any modules. Pass an empty list if the workflows
+            were already loaded by the time this class is instantiated.
+
+        :raises lambda_builders.exceptions.WorkflowNotFoundError: If a workflow for given capabilities is not found
+        """
+
+        # Load defaults if necessary. We check for `None` explicitly because callers could pass an empty list
+        # if they do not want to load any modules. This supports the case where workflows are already loaded and
+        # don't need to be loaded again.
+        self.supported_workflows = _SUPPORTED_WORKFLOWS if supported_workflows is None else supported_workflows
+
+        for workflow_module in self.supported_workflows:
+            LOG.debug("Loading workflow module '%s'", workflow_module)
+
+            # If a module is already loaded, this call is pretty much a no-op. So it is okay to keep loading again.
+            importlib.import_module(workflow_module)
+
+        self.capability = Capability(language=language,
+                                     dependency_manager=dependency_manager,
+                                     application_framework=application_framework)
+        self.selected_workflow_cls = get_workflow(self.capability)
+        LOG.debug("Found workflow '%s' to support capabilities '%s'", self.selected_workflow_cls.NAME, self.capability)
+
+    def build(self, source_dir, artifacts_dir, scratch_dir, manifest_path,
+              runtime=None, optimizations=None, options=None):
+        """
+        Actually build the code by running workflows
+
+        :type source_dir: str
+        :param source_dir:
+            Path to a folder containing the source code
+
+        :type artifacts_dir: str
+        :param artifacts_dir:
+            Path to a folder where the built artifacts should be placed
+
+        :type scratch_dir: str
+        :param scratch_dir:
+            Path to a directory that the workflow can use as scratch space. Workflows are expected to use this directory
+            to write temporary files instead of ``/tmp`` or other OS-specific temp directories.
+
+        :type manifest_path: str
+        :param manifest_path:
+            Path to the dependency manifest
+
+        :type runtime: str
+        :param runtime:
+            Optional, name of the AWS Lambda runtime that you are building for. This is sent to the builder for
+            informational purposes.
+
+        :type optimizations: dict
+        :param optimizations:
+            Optional dictionary of optimization flags to pass to the build action. **Not supported**.
+
+        :type options: dict
+        :param options:
+            Optional dictionary of options ot pass to build action. **Not supported**.
+        """
+
+        workflow = self.selected_workflow_cls(source_dir,
+                                              artifacts_dir,
+                                              scratch_dir,
+                                              manifest_path,
+                                              runtime=runtime,
+                                              optimizations=optimizations,
+                                              options=options)
+
+        return workflow.run()
+
+    def _clear_workflows(self):
+        DEFAULT_REGISTRY.clear()

aws_lambda_builders/exceptions.py

@@ -15,15 +15,23 @@ class UnsupportedManifestError(LambdaBuilderError):
     MESSAGE = "A builder for the given capabilities '{capabilities}' was not found"
 
 
-class WorkflowFailed(LambdaBuilderError):
+class WorkflowNotFoundError(LambdaBuilderError):
+    """
+    Raised when a workflow matching the given capabilities was not found
+    """
+    MESSAGE = "Unable to find a workflow matching given capability: " \
+              "{language}, {dependency_manager}, {application_framework}"
+
+
+class WorkflowFailedError(LambdaBuilderError):
     """
     Raised when the build failed, for well-known cases
     """
-    MESSAGE = "'{workflow_name}' workflow failed: {reason}"
+    MESSAGE = "Workflow='{workflow_name}',Action='{action_name}' failed: {reason}"
 
 
-class WorkflowError(LambdaBuilderError):
+class WorkflowUnknownError(LambdaBuilderError):
     """
     Raised when the build ran into an unexpected error
     """
-    MESSAGE = "'{workflow_name}' workflow ran into an error: {reason}"
+    MESSAGE = "Workflow='{workflow_name}',Action='{action_name}' ran into an error: {reason}"