From 45e75b649cf97df25f1a1b7468aa0da3295c11ea Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Fri, 13 Sep 2019 15:01:12 -0700 Subject: [PATCH 01/49] WIP: Interactive Init Command This is populating new commands and changes some of the relevant documentation. This is very much a WIP and not ready to merge. --- samcli/commands/init/__init__.py | 37 +++++++++++++++++++------------- samcli/local/init/__init__.py | 13 ++++++----- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 9164ddeb90..3f4ec6a438 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -20,6 +20,12 @@ @click.command( "init", short_help="Init an AWS SAM application.", context_settings=dict(help_option_names=[u"-h", u"--help"]) ) +@click.option( + "--no-interactive", + is_flag=True, + default=False, + help="Disable interactive prompting for values, and fail if any required values are missing.", +) @click.option("-l", "--location", help="Template location (git, mercurial, http(s), zip, path)") @click.option( "-r", "--runtime", type=click.Choice(INIT_RUNTIMES), default=DEFAULT_RUNTIME, help="Lambda Runtime of your app" @@ -40,10 +46,15 @@ default=False, help="Disable prompting and accept default values defined template config", ) +@click.option( + "--application-template", + default=None, + help="If using a managed AWS SAM CLI application template, provide its identifier." +) @common_options @pass_context @track_command -def cli(ctx, location, runtime, dependency_manager, output_dir, name, no_input): +def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, app_template): """ \b Initialize a serverless application with a SAM template, folder structure for your Lambda functions, connected to an event source such as APIs, @@ -57,41 +68,37 @@ def cli(ctx, location, runtime, dependency_manager, output_dir, name, no_input): Common usage: \b - Initializes a new SAM project using Python 3.6 default template runtime - \b - $ sam init --runtime python3.6 - \b - Initializes a new SAM project using Java 8 and Gradle dependency manager + Starts an interactive prompt process to initialize a new project: \b - $ sam init --runtime java8 --dependency-manager gradle + $ sam init \b Initializes a new SAM project using custom template in a Git/Mercurial repository \b # gh being expanded to github url - $ sam init --location gh:aws-samples/cookiecutter-aws-sam-python + $ sam init --location gh:aws-samples/cookiecutter-aws-sam-python --name python-sam-app \b - $ sam init --location git+ssh://git@github.com/aws-samples/cookiecutter-aws-sam-python.git + $ sam init --location git+ssh://git@github.com/aws-samples/cookiecutter-aws-sam-python.git --name python-sam-app \b - $ sam init --location hg+ssh://hg@bitbucket.org/repo/template-name + $ sam init --location hg+ssh://hg@bitbucket.org/repo/template-name --name python-sam-app \b Initializes a new SAM project using custom template in a Zipfile \b - $ sam init --location /path/to/template.zip + $ sam init --location /path/to/template.zip --name sam-app \b - $ sam init --location https://example.com/path/to/template.zip + $ sam init --location https://example.com/path/to/template.zip --name sam-app \b Initializes a new SAM project using custom template in a local path \b - $ sam init --location /path/to/template/folder + $ sam init --location /path/to/template/folder --name sam-app """ # All logic must be implemented in the `do_cli` method. This helps ease unit tests - do_cli(ctx, location, runtime, dependency_manager, output_dir, name, no_input) # pragma: no cover + do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, app_template) # pragma: no cover -def do_cli(ctx, location, runtime, dependency_manager, output_dir, name, no_input): +def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, app_template): """ Implementation of the ``cli`` method, just separated out for unit testing purposes """ diff --git a/samcli/local/init/__init__.py b/samcli/local/init/__init__.py index 6a497b6280..c5fe30a785 100644 --- a/samcli/local/init/__init__.py +++ b/samcli/local/init/__init__.py @@ -14,7 +14,7 @@ def generate_project( - location=None, runtime="nodejs10.x", dependency_manager=None, output_dir=".", name="sam-sample-app", no_input=False + location=None, runtime=None, dependency_manager=None, output_dir=".", name=None, no_input=False ): """Generates project using cookiecutter and options given @@ -27,16 +27,15 @@ def generate_project( location: Path, optional Git, HTTP, Local path or Zip containing cookiecutter template (the default is None, which means no custom template) - runtime: str, optional - Lambda Runtime (the default is "nodejs", which creates a nodejs project) + runtime: str + Lambda Runtime dependency_manager: str, optional - Dependency Manager for the Lambda Runtime Project(the default is "npm" for a "nodejs" Lambda runtime) + Dependency Manager for the Lambda Runtime Project output_dir: str, optional Output directory where project should be generated (the default is ".", which implies current folder) - name: str, optional + name: str Name of the project - (the default is "sam-sample-app", which implies a project named sam-sample-app will be created) no_input : bool, optional Whether to prompt for input or to accept default values (the default is False, which prompts the user for values it doesn't know for baking) @@ -68,7 +67,7 @@ def generate_project( params["extra_context"] = {"project_name": name, "runtime": runtime} params["no_input"] = True LOG.debug("Parameters dict updated with project name as extra_context") - LOG.debug("%s", params) + LOG.dpebug("%s", params) try: LOG.debug("Baking a new template with cookiecutter with all parameters") From 33eb157bb050736d122784d0b4eca7051370dc6a Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 24 Sep 2019 16:14:52 -0700 Subject: [PATCH 02/49] WIP: Mutually Exclusive and Required Params --- samcli/commands/init/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 3f4ec6a438..76a19f1dc2 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -105,6 +105,16 @@ def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_di LOG.debug("Init command") click.secho("[+] Initializing project structure...", fg="green") + # check for mutually exclusive parameters and fail + if app_template and location: + raise "You must not provide both --application-template and --location" + + # check for required parameters + if runtime and name and (app_template or location): + # proceed to generation + else: + # proceed to interactive state machine + no_build_msg = """ Project generated: {output_dir}/{name} From 5aba8292aef1b9616625a01f9c4debcbb2501132 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Fri, 27 Sep 2019 14:31:00 -0700 Subject: [PATCH 03/49] WIP: Stubbed Out Interactive Flow This actually covers all values we need to prompt for, but is in dire need of a refactoring, and the GitHub clone logic needs to be implemented. --- samcli/commands/init/__init__.py | 138 +++++++++++++----------- samcli/local/common/runtime_template.py | 16 +++ samcli/local/init/__init__.py | 6 +- 3 files changed, 95 insertions(+), 65 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 76a19f1dc2..4e8068d771 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -8,12 +8,11 @@ from samcli.cli.main import pass_context, common_options from samcli.commands.exceptions import UserException -from samcli.local.common.runtime_template import INIT_RUNTIMES, SUPPORTED_DEP_MANAGERS, DEFAULT_RUNTIME +from samcli.local.common.runtime_template import INIT_RUNTIMES, RUNTIME_TO_DEPENDENCY_MANAGERS, SUPPORTED_DEP_MANAGERS from samcli.local.init import generate_project from samcli.local.init.exceptions import GenerateProjectFailedError from samcli.lib.telemetry.metrics import track_command - LOG = logging.getLogger(__name__) @@ -27,9 +26,7 @@ help="Disable interactive prompting for values, and fail if any required values are missing.", ) @click.option("-l", "--location", help="Template location (git, mercurial, http(s), zip, path)") -@click.option( - "-r", "--runtime", type=click.Choice(INIT_RUNTIMES), default=DEFAULT_RUNTIME, help="Lambda Runtime of your app" -) +@click.option("-r", "--runtime", type=click.Choice(INIT_RUNTIMES), help="Lambda Runtime of your app") @click.option( "-d", "--dependency-manager", @@ -38,8 +35,8 @@ help="Dependency manager of your Lambda runtime", required=False, ) -@click.option("-o", "--output-dir", default=".", type=click.Path(), help="Where to output the initialized app into") -@click.option("-n", "--name", default="sam-app", help="Name of your project to be generated as a folder") +@click.option("-o", "--output-dir", type=click.Path(), help="Where to output the initialized app into") +@click.option("-n", "--name", help="Name of your project to be generated as a folder") @click.option( "--no-input", is_flag=True, @@ -49,72 +46,92 @@ @click.option( "--application-template", default=None, - help="If using a managed AWS SAM CLI application template, provide its identifier." + help="If using a managed AWS SAM CLI application template, provide its identifier.", ) @common_options @pass_context @track_command -def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, app_template): - """ \b - Initialize a serverless application with a SAM template, folder - structure for your Lambda functions, connected to an event source such as APIs, - S3 Buckets or DynamoDB Tables. This application includes everything you need to - get started with serverless and eventually grow into a production scale application. - \b - This command can initialize a boilerplate serverless app. If you want to create your own - template as well as use a custom location please take a look at our official documentation. - - \b - Common usage: - - \b - Starts an interactive prompt process to initialize a new project: - \b - $ sam init - \b - Initializes a new SAM project using custom template in a Git/Mercurial repository - \b - # gh being expanded to github url - $ sam init --location gh:aws-samples/cookiecutter-aws-sam-python --name python-sam-app - \b - $ sam init --location git+ssh://git@github.com/aws-samples/cookiecutter-aws-sam-python.git --name python-sam-app - \b - $ sam init --location hg+ssh://hg@bitbucket.org/repo/template-name --name python-sam-app - - \b - Initializes a new SAM project using custom template in a Zipfile - \b - $ sam init --location /path/to/template.zip --name sam-app - \b - $ sam init --location https://example.com/path/to/template.zip --name sam-app - - \b - Initializes a new SAM project using custom template in a local path - \b - $ sam init --location /path/to/template/folder --name sam-app - - """ - # All logic must be implemented in the `do_cli` method. This helps ease unit tests - do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, app_template) # pragma: no cover +def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, application_template): + do_cli( + ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, application_template + ) # pragma: no cover def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, app_template): - """ - Implementation of the ``cli`` method, just separated out for unit testing purposes - """ - LOG.debug("Init command") - click.secho("[+] Initializing project structure...", fg="green") - # check for mutually exclusive parameters and fail if app_template and location: raise "You must not provide both --application-template and --location" # check for required parameters - if runtime and name and (app_template or location): - # proceed to generation + if name and ((runtime and app_template) or location): + _do_generate(location, runtime, dependency_manager, output_dir, name, no_input, app_template) + elif no_interactive: + # raise error - this message is a slight mess, and hard to read + error_msg = """ +ERROR: Missing required parameters, with --no-interactive set. + +Must provide at one of the following required parameter combinations: + --name and --application-template and --runtime + --name and --location + +You can also re-run without the --no-interactive flag to be prompted for required values. + """ + raise UserException(error_msg) else: - # proceed to interactive state machine - + # proceed to interactive state machine, which will call _do_generate + _do_interactive(location, runtime, dependency_manager, output_dir, name, no_input, app_template) + + +def _do_interactive(location, runtime, dependency_manager, output_dir, name, no_input, app_template): + if not name: + name = click.prompt("Project Name", type=str) + if not location: + if not runtime: + # TODO: Better output than click default choices. + runtime = click.prompt("Runtime", type=click.Choice(INIT_RUNTIMES)) + # TODO: Only fetch this for default app templates? Don't want to give a false impression of all templates being + # available for multiple dependency managers. + if not dependency_manager: + valid_dep_managers = RUNTIME_TO_DEPENDENCY_MANAGERS.get(runtime) + if valid_dep_managers is None: + dependency_manager = None + else: + dependency_manager = click.prompt( + "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] + ) + if not (location or app_template): + # pull app template choices from github and display + # with alternate option to pick custom, which prompts location + templates_folder = _clone_app_templates + app_template_options = _get_manifest_options(runtime, templates_folder) + app_template_options.append("Custom Template") + # this should display with number codes for selection - + at_choice = click.prompt("Template", type=click.Choice(app_template_options)) + if at_choice is "Custom Template": + print("Doc Link for Valid Options: ***") + location = click.prompt("App Template Location", type=str) + else: + # location is in the local filepath - send extra context to cookiecutter + location = _get_app_template_folder(templates_folder, at_choice) + if not output_dir: + output_dir = click.prompt("Output Directory", type=click.Path(), default=".") + _do_generate(location, runtime, dependency_manager, output_dir, name, no_input, app_template) + + +def _clone_app_templates(): + return None + + +def _get_app_template_folder(templates_folder, app_template_choice): + return None + + +def _get_manifest_options(runtime, templates_folder): + return ["Option 1", "Option 2"] + + +def _do_generate(location, runtime, dependency_manager, output_dir, name, no_input, app_template): + # Todo: This needs to handle app_template no_build_msg = """ Project generated: {output_dir}/{name} @@ -151,7 +168,6 @@ def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_di "ruby2.5", ) next_step_msg = no_build_msg if runtime in no_build_step_required else build_msg - try: generate_project(location, runtime, dependency_manager, output_dir, name, no_input) if not location: diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index b76c0ae9a9..ff187e3092 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -78,6 +78,22 @@ ], } +RUNTIME_TO_DEPENDENCY_MANAGERS = { + "python3.7": ["pip"], + "python3.6": ["pip"], + "python2.7": ["pip"], + "ruby2.5": ["bundler"], + "nodejs10.x": ["npm"], + "nodejs8.10": ["npm"], + "nodejs6.10": ["npm"], + "dotnetcore2.1": ["cli-package"], + "dotnetcore2.0": ["cli-package"], + "dotnetcore1.0": ["cli-package"], + "go1.x": None, + "java8": ["maven", "gradle"], +} + + SUPPORTED_DEP_MANAGERS = set( [ c["dependency_manager"] diff --git a/samcli/local/init/__init__.py b/samcli/local/init/__init__.py index c5fe30a785..3334e9c6c9 100644 --- a/samcli/local/init/__init__.py +++ b/samcli/local/init/__init__.py @@ -13,9 +13,7 @@ LOG = logging.getLogger(__name__) -def generate_project( - location=None, runtime=None, dependency_manager=None, output_dir=".", name=None, no_input=False -): +def generate_project(location=None, runtime=None, dependency_manager=None, output_dir=".", name=None, no_input=False): """Generates project using cookiecutter and options given Generate project scaffolds a project using default templates if user @@ -67,7 +65,7 @@ def generate_project( params["extra_context"] = {"project_name": name, "runtime": runtime} params["no_input"] = True LOG.debug("Parameters dict updated with project name as extra_context") - LOG.dpebug("%s", params) + LOG.debug("%s", params) try: LOG.debug("Baking a new template with cookiecutter with all parameters") From 01206f1fcdf061670509fd28f960dd6ea73429e6 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Wed, 2 Oct 2019 15:15:24 -0700 Subject: [PATCH 04/49] WIP: Working Flow The flow works, most edge cases covered. Still have some development features that need to be pulled out/resolved. --- samcli/commands/init/__init__.py | 137 +++--------------- samcli/commands/init/init_generator.py | 52 +++++++ samcli/commands/init/init_templates.py | 108 ++++++++++++++ samcli/commands/init/interactive_init_flow.py | 37 +++++ samcli/local/common/runtime_template.py | 4 +- samcli/local/init/__init__.py | 24 +-- 6 files changed, 232 insertions(+), 130 deletions(-) create mode 100644 samcli/commands/init/init_generator.py create mode 100644 samcli/commands/init/init_templates.py create mode 100644 samcli/commands/init/interactive_init_flow.py diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 4e8068d771..999b6dc14f 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -2,16 +2,20 @@ """ Init command to scaffold a project app from a template """ +import itertools +import json import logging +import os +import subprocess import click -from samcli.cli.main import pass_context, common_options +from samcli.cli.main import pass_context, common_options, global_cfg from samcli.commands.exceptions import UserException -from samcli.local.common.runtime_template import INIT_RUNTIMES, RUNTIME_TO_DEPENDENCY_MANAGERS, SUPPORTED_DEP_MANAGERS -from samcli.local.init import generate_project -from samcli.local.init.exceptions import GenerateProjectFailedError +from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS from samcli.lib.telemetry.metrics import track_command +from samcli.commands.init.init_generator import do_generate +from samcli.commands.init.interactive_init_flow import do_interactive LOG = logging.getLogger(__name__) @@ -26,7 +30,7 @@ help="Disable interactive prompting for values, and fail if any required values are missing.", ) @click.option("-l", "--location", help="Template location (git, mercurial, http(s), zip, path)") -@click.option("-r", "--runtime", type=click.Choice(INIT_RUNTIMES), help="Lambda Runtime of your app") +@click.option("-r", "--runtime", type=click.Choice(RUNTIMES), help="Lambda Runtime of your app") @click.option( "-d", "--dependency-manager", @@ -43,136 +47,29 @@ default=False, help="Disable prompting and accept default values defined template config", ) -@click.option( - "--application-template", - default=None, - help="If using a managed AWS SAM CLI application template, provide its identifier.", -) @common_options @pass_context @track_command -def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, application_template): - do_cli( - ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, application_template - ) # pragma: no cover +def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input): + do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input) # pragma: no cover -def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input, app_template): - # check for mutually exclusive parameters and fail - if app_template and location: - raise "You must not provide both --application-template and --location" - +def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input): # check for required parameters - if name and ((runtime and app_template) or location): - _do_generate(location, runtime, dependency_manager, output_dir, name, no_input, app_template) + if name and (runtime or location): + do_generate(location, runtime, dependency_manager, output_dir, name, no_input, None) elif no_interactive: # raise error - this message is a slight mess, and hard to read error_msg = """ ERROR: Missing required parameters, with --no-interactive set. Must provide at one of the following required parameter combinations: - --name and --application-template and --runtime + --name and --runtime --name and --location You can also re-run without the --no-interactive flag to be prompted for required values. """ raise UserException(error_msg) else: - # proceed to interactive state machine, which will call _do_generate - _do_interactive(location, runtime, dependency_manager, output_dir, name, no_input, app_template) - - -def _do_interactive(location, runtime, dependency_manager, output_dir, name, no_input, app_template): - if not name: - name = click.prompt("Project Name", type=str) - if not location: - if not runtime: - # TODO: Better output than click default choices. - runtime = click.prompt("Runtime", type=click.Choice(INIT_RUNTIMES)) - # TODO: Only fetch this for default app templates? Don't want to give a false impression of all templates being - # available for multiple dependency managers. - if not dependency_manager: - valid_dep_managers = RUNTIME_TO_DEPENDENCY_MANAGERS.get(runtime) - if valid_dep_managers is None: - dependency_manager = None - else: - dependency_manager = click.prompt( - "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] - ) - if not (location or app_template): - # pull app template choices from github and display - # with alternate option to pick custom, which prompts location - templates_folder = _clone_app_templates - app_template_options = _get_manifest_options(runtime, templates_folder) - app_template_options.append("Custom Template") - # this should display with number codes for selection - - at_choice = click.prompt("Template", type=click.Choice(app_template_options)) - if at_choice is "Custom Template": - print("Doc Link for Valid Options: ***") - location = click.prompt("App Template Location", type=str) - else: - # location is in the local filepath - send extra context to cookiecutter - location = _get_app_template_folder(templates_folder, at_choice) - if not output_dir: - output_dir = click.prompt("Output Directory", type=click.Path(), default=".") - _do_generate(location, runtime, dependency_manager, output_dir, name, no_input, app_template) - - -def _clone_app_templates(): - return None - - -def _get_app_template_folder(templates_folder, app_template_choice): - return None - - -def _get_manifest_options(runtime, templates_folder): - return ["Option 1", "Option 2"] - - -def _do_generate(location, runtime, dependency_manager, output_dir, name, no_input, app_template): - # Todo: This needs to handle app_template - no_build_msg = """ -Project generated: {output_dir}/{name} - -Steps you can take next within the project folder -=================================================== -[*] Invoke Function: sam local invoke HelloWorldFunction --event event.json -[*] Start API Gateway locally: sam local start-api -""".format( - output_dir=output_dir, name=name - ) - - build_msg = """ -Project generated: {output_dir}/{name} - -Steps you can take next within the project folder -=================================================== -[*] Install dependencies -[*] Invoke Function: sam local invoke HelloWorldFunction --event event.json -[*] Start API Gateway locally: sam local start-api -""".format( - output_dir=output_dir, name=name - ) - - no_build_step_required = ( - "python", - "python3.7", - "python3.6", - "python2.7", - "nodejs", - "nodejs4.3", - "nodejs6.10", - "nodejs8.10", - "nodejs10.x", - "ruby2.5", - ) - next_step_msg = no_build_msg if runtime in no_build_step_required else build_msg - try: - generate_project(location, runtime, dependency_manager, output_dir, name, no_input) - if not location: - click.secho(next_step_msg, bold=True) - click.secho("Read {name}/README.md for further instructions\n".format(name=name), bold=True) - click.secho("[*] Project initialization is now complete", fg="green") - except GenerateProjectFailedError as e: - raise UserException(str(e)) + # proceed to interactive state machine, which will call do_generate + do_interactive(location, runtime, dependency_manager, output_dir, name, no_input) diff --git a/samcli/commands/init/init_generator.py b/samcli/commands/init/init_generator.py new file mode 100644 index 0000000000..cf49e8683d --- /dev/null +++ b/samcli/commands/init/init_generator.py @@ -0,0 +1,52 @@ +import click + +from samcli.commands.exceptions import UserException +from samcli.local.init import generate_project +from samcli.local.init.exceptions import GenerateProjectFailedError + + +def do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context): + no_build_msg = """ +Project generated: {output_dir}/{name} + +Steps you can take next within the project folder +=================================================== +[*] Invoke Function: sam local invoke HelloWorldFunction --event event.json +[*] Start API Gateway locally: sam local start-api +""".format( + output_dir=output_dir, name=name + ) + + build_msg = """ +Project generated: {output_dir}/{name} + +Steps you can take next within the project folder +=================================================== +[*] Install dependencies +[*] Invoke Function: sam local invoke HelloWorldFunction --event event.json +[*] Start API Gateway locally: sam local start-api +""".format( + output_dir=output_dir, name=name + ) + + no_build_step_required = ( + "python", + "python3.7", + "python3.6", + "python2.7", + "nodejs", + "nodejs4.3", + "nodejs6.10", + "nodejs8.10", + "nodejs10.x", + "ruby2.5", + ) + next_step_msg = no_build_msg if runtime in no_build_step_required else build_msg + try: + generate_project(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) + if not location: + click.secho(next_step_msg, bold=True) + click.secho("Read {name}/README.md for further instructions\n".format(name=name), bold=True) + click.secho("[*] Project initialization is now complete", fg="green") + except GenerateProjectFailedError as e: + raise UserException(str(e)) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py new file mode 100644 index 0000000000..c58174f7b1 --- /dev/null +++ b/samcli/commands/init/init_templates.py @@ -0,0 +1,108 @@ +""" +Manages the set of application templates. +""" + +import click +import itertools +import json +import os +import shutil +import subprocess + +from pathlib import Path # must come after Py2.7 deprecation + +from samcli.cli.main import global_cfg +from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING + + +class InitTemplates: + def __init__(self): + # self._repo_url = "https://github.com/awslabs/aws-sam-cli-app-templates.git" + # testing only, delete this and uncomment the above line before shipping + self._repo_url = "git@github.com:awslabs/aws-sam-cli-app-templates.git" + self._repo_name = "aws-sam-cli-app-templates" + self.repo_path = None + self.clone_attempted = False + + def prompt_for_location(self, runtime, dependency_manager): + options = self.init_options(runtime, dependency_manager) + choices = map(str, range(1, len(options) + 1)) + choice_num = 1 + for o in options: + if o.get("displayName") is not None: + print (choice_num, "-", o.get("displayName")) + else: + print ( + choice_num, + "- Default Template for runtime", + runtime, + "with dependency manager", + dependency_manager, + ) + choice_num = choice_num + 1 + choice = click.prompt("Template Selection", type=click.Choice(choices), show_choices=False) + template_md = options[int(choice) - 1] # zero index + if template_md.get("init_location") is not None: + return template_md["init_location"] + elif template_md.get("directory") is not None: + return os.path.join(self.repo_path, template_md["directory"]) + else: + raise UserException("Invalid template. This should not be possible, please raise an issue.") + + def init_options(self, runtime, dependency_manager): + if self.clone_attempted is False: + self._clone_repo() + if self.repo_path is None: + return self._init_options_from_bundle(runtime, dependency_manager) + else: + return self._init_options_from_manifest(runtime, dependency_manager) + + def _init_options_from_manifest(self, runtime, dependency_manager): + manifest_path = os.path.join(self.repo_path, "manifest.json") + with open(str(manifest_path)) as fp: + body = fp.read() + manifest_body = json.loads(body) + templates = manifest_body.get(runtime) + if templates is None: + # Fallback to bundled templates + return self._init_options_from_bundle(runtime, dependency_manager) + else: + return templates + + def _init_options_from_bundle(self, runtime, dependency_manager): + for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): + if runtime in mapping["runtimes"] or any([r.startswith(runtime) for r in mapping["runtimes"]]): + if not dependency_manager or dependency_manager == mapping["dependency_manager"]: + return [mapping] + msg = "Lambda Runtime {} and dependency manager {} does not have an available initialization template.".format( + runtime, dependency_manager + ) + raise UserException(msg) + + def _clone_repo(self): + shared_dir = global_cfg.config_dir + expected_path = os.path.normpath(os.path.join(shared_dir, self._repo_name)) + if self._should_clone_repo(expected_path): + try: + subprocess.check_output(["git", "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT) + # check for repo in path? + self.repo_path = expected_path + except subprocess.CalledProcessError as clone_error: + output = clone_error.output.decode("utf-8") + if "not found" in output.lower(): + print ("WARN: Could not clone app template repo.") + # do we ever want to hard fail? + self._clone_attempted = True + + def _should_clone_repo(self, expected_path): + path = Path(expected_path) + if path.exists(): + overwrite = click.confirm("Init templates exist on disk. Do you wish to update?") + if overwrite: + # possible alternative: pull down first, THEN delete old version - safer if there is a problem + shutil.rmtree(expected_path) # fail hard if there is an issue + return True + self.repo_path = expected_path + return False + else: + return True diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py new file mode 100644 index 0000000000..60de6c5bc6 --- /dev/null +++ b/samcli/commands/init/interactive_init_flow.py @@ -0,0 +1,37 @@ +import click + +from samcli.local.common.runtime_template import RUNTIMES, RUNTIME_TO_DEPENDENCY_MANAGERS +from samcli.commands.init.init_generator import do_generate +from samcli.commands.init.init_templates import InitTemplates + + +def do_interactive(location, runtime, dependency_manager, output_dir, name, no_input): + extra_context = None + if not location: + print( + "1 - Use a Managed Application Template\n2 - Provide a Custom Location" + ) + location_opt_choice = click.prompt("Location Choice", type=click.Choice(['1', '2']), show_choices=False) + if location_opt_choice == '2': + location = click.prompt("Template location (git, mercurial, http(s), zip, path)", type=str) + else: + if not name: + name = click.prompt("Project Name", type=str) + if not runtime: + # TODO: Better output than click default choices. + runtime = click.prompt("Runtime", type=click.Choice(RUNTIMES)) + if not dependency_manager: + valid_dep_managers = RUNTIME_TO_DEPENDENCY_MANAGERS.get(runtime) + if valid_dep_managers is None: + dependency_manager = None + else: + dependency_manager = click.prompt( + "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] + ) + templates = InitTemplates() + location = templates.prompt_for_location(runtime, dependency_manager) + no_input = True # because we specified the template ourselves + extra_context = {"project_name": name, "runtime": runtime} + if not output_dir: + output_dir = click.prompt("Output Directory", type=click.Path(), default=".") + do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index ff187e3092..da8dd9fd85 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -48,7 +48,7 @@ ], "dotnet": [ { - "runtimes": ["dotnetcore2.1", "dotnetcore2.0", "dotnetcore1.0", "dotnetcore"], + "runtimes": ["dotnetcore2.1", "dotnetcore2.0", "dotnetcore1.0"], "dependency_manager": "cli-package", "init_location": os.path.join(_templates, "cookiecutter-aws-sam-hello-dotnet"), "build": True, @@ -101,9 +101,11 @@ if c["dependency_manager"] ] ) + RUNTIMES = set( itertools.chain(*[c["runtimes"] for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))]) ) + INIT_RUNTIMES = RUNTIMES.union(RUNTIME_DEP_TEMPLATE_MAPPING.keys()) # NOTE(TheSriram): Default Runtime Choice when runtime is not chosen diff --git a/samcli/local/init/__init__.py b/samcli/local/init/__init__.py index 3334e9c6c9..3227d498ec 100644 --- a/samcli/local/init/__init__.py +++ b/samcli/local/init/__init__.py @@ -13,7 +13,9 @@ LOG = logging.getLogger(__name__) -def generate_project(location=None, runtime=None, dependency_manager=None, output_dir=".", name=None, no_input=False): +def generate_project( + location=None, runtime=None, dependency_manager=None, output_dir=".", name=None, no_input=False, extra_context=None +): """Generates project using cookiecutter and options given Generate project scaffolds a project using default templates if user @@ -46,18 +48,22 @@ def generate_project(location=None, runtime=None, dependency_manager=None, outpu template = None - for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): - if runtime in mapping["runtimes"] or any([r.startswith(runtime) for r in mapping["runtimes"]]): - if not dependency_manager or dependency_manager == mapping["dependency_manager"]: - template = mapping["init_location"] - break + if runtime: + for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): + if runtime in mapping["runtimes"] or any([r.startswith(runtime) for r in mapping["runtimes"]]): + if not dependency_manager or dependency_manager == mapping["dependency_manager"]: + template = mapping["init_location"] + break - if not template: - msg = "Lambda Runtime {} does not support dependency manager: {}".format(runtime, dependency_manager) - raise GenerateProjectFailedError(project=name, provider_error=msg) + if not template: + msg = "Lambda Runtime {} does not support dependency manager: {}".format(runtime, dependency_manager) + raise GenerateProjectFailedError(project=name, provider_error=msg) params = {"template": location if location else template, "output_dir": output_dir, "no_input": no_input} + if extra_context: + params["extra_context"] = extra_context + LOG.debug("Parameters dict created with input given") LOG.debug("%s", params) From 928aba6a1fd870943981d71be9a6eb45131e4589 Mon Sep 17 00:00:00 2001 From: Trenton Lipscomb Date: Mon, 7 Oct 2019 10:33:49 -0700 Subject: [PATCH 05/49] Adds SageMaker Ground Truth Lambda events Adds events for Lambda functions for: 1. PreHumanTask 2. AnnotationConsolidation --- .../event-mapping.json | 58 ++++++++++++++++++- .../sagemaker/AnnotationConsolidation.json | 9 +++ .../events/sagemaker/PreHumanTask.json | 7 +++ 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json create mode 100644 samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json diff --git a/samcli/commands/local/lib/generated_sample_events/event-mapping.json b/samcli/commands/local/lib/generated_sample_events/event-mapping.json index b5c6ba4cba..60a6892b9a 100644 --- a/samcli/commands/local/lib/generated_sample_events/event-mapping.json +++ b/samcli/commands/local/lib/generated_sample_events/event-mapping.json @@ -849,6 +849,62 @@ } } }, + "sagemaker": { + "ground-truth-pre-human": { + "filename": "PreHumanTask", + "help": "Generates a SageMaker Ground Truth PreHumanTask request", + "tags": { + "region": { + "default": "us-east-1" + }, + "partition": { + "default": "aws" + }, + "account-id": { + "default": "123456789012" + }, + "bucket": { + "default": "sagemakerexample" + }, + "data_object_key": { + "default": "object_to_annotate.jpg" + }, + "labeling_job_name": { + "default": "example-job" + } + } + }, + "ground-truth-annotation-consolidation": { + "filename": "AnnotationConsolidation", + "help": "Generates a SageMaker Ground Truth AnnotationConsolidation request", + "tags": { + "region": { + "default": "us-east-1" + }, + "partition": { + "default": "aws" + }, + "account-id": { + "default": "123456789012" + }, + "labeling_job_name": { + "default": "example-job" + }, + "label_attribute_name": { + "default": "example-attribute" + }, + "bucket": { + "default": "sagemakerexample" + }, + "execution_role": { + "default": "sagemaker-role" + }, + "iteration_object_timestamp": { + "default": "iteration-1/0/2019-09-06_18:35:03" + } + } + } + }, "ses": { "email-receiving": { "filename": "SesEmailReceiving", @@ -926,4 +982,4 @@ "tags": {} } } -} \ No newline at end of file +} diff --git a/samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json new file mode 100644 index 0000000000..083bc76583 --- /dev/null +++ b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json @@ -0,0 +1,9 @@ +{ + "version": "2018-10-16", + "labelingJobArn": "arn:{{{partition}}}:sagemaker:{{{region}}}:{{{account_id}}}:labeling-job/{{{labeling_job_name}}}", + "labelAttributeName": "{{{label_attribute_name}}}", + "roleArn" : "aws:{{{partition}}}:iam::{{account_id}}:role/{{{execution_role}}}", + "payload": { + "s3Uri": "s3://{{{bucket}}}/{{{labeling_job_name}}}/annotations/worker_response/{{{iteration_object_timestamp}}}.json" + } + } diff --git a/samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json new file mode 100644 index 0000000000..079e04435c --- /dev/null +++ b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json @@ -0,0 +1,7 @@ +{ + "version": "2018-10-16", + "labelingJobArn": "arn:{{{partition}}}:sagemaker:{{{region}}}:{{{account_id}}}:labeling-job/{{{labeling_job_name}}}", + "dataObject" : { + "source-ref": "s3://{{{bucket}}}/{{{data_object_key}}}" + } +} From 911dc86120d0f2a49496a0f3d51281d780c42069 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 7 Oct 2019 14:04:07 -0700 Subject: [PATCH 06/49] WIP: Working Init Flow Handles most cases, still need to clean out some debug logic and write full test suite. --- samcli/commands/init/__init__.py | 43 ++++++++++++++----- samcli/commands/init/init_templates.py | 39 ++++++++++++++--- samcli/commands/init/interactive_init_flow.py | 25 +++++++---- 3 files changed, 81 insertions(+), 26 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 999b6dc14f..edf1375870 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -15,6 +15,7 @@ from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS from samcli.lib.telemetry.metrics import track_command from samcli.commands.init.init_generator import do_generate +from samcli.commands.init.init_templates import InitTemplates from samcli.commands.init.interactive_init_flow import do_interactive LOG = logging.getLogger(__name__) @@ -41,6 +42,10 @@ ) @click.option("-o", "--output-dir", type=click.Path(), help="Where to output the initialized app into") @click.option("-n", "--name", help="Name of your project to be generated as a folder") +@click.option( + "--app-template", + help="Identifier of the managed application template you want to use. If not sure, call 'sam init' without options for an interactive workflow.", +) @click.option( "--no-input", is_flag=True, @@ -50,26 +55,44 @@ @common_options @pass_context @track_command -def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input): - do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input) # pragma: no cover +def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input): + do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input) # pragma: no cover -def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, no_input): +def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input): + # check for mutually exclusive parameters + if location and app_template: + msg = """ +You must not provide both the --location and --app-template parameters. + +You can run 'sam init' without any options for an interactive initialization flow, or you can provide one of the following required parameter combinations: + --name and --runtime and --app-template + --location + """ + raise UserException(msg) # check for required parameters - if name and (runtime or location): - do_generate(location, runtime, dependency_manager, output_dir, name, no_input, None) + if location or (name and runtime and dependency_manager and app_template): + # need to turn app_template into a location before we generate + extra_context = None + if app_template: + templates = InitTemplates(no_interactive) + location = templates.location_from_app_template(runtime, dependency_manager, app_template) + no_input = True + extra_context = {"project_name": name, "runtime": runtime} + if not output_dir: + output_dir = "." # default - should I lift this to overall options and seed default? + do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) elif no_interactive: - # raise error - this message is a slight mess, and hard to read error_msg = """ ERROR: Missing required parameters, with --no-interactive set. -Must provide at one of the following required parameter combinations: - --name and --runtime - --name and --location +Must provide one of the following required parameter combinations: + --name and --runtime and --dependency-manager and --app-template + --location You can also re-run without the --no-interactive flag to be prompted for required values. """ raise UserException(error_msg) else: # proceed to interactive state machine, which will call do_generate - do_interactive(location, runtime, dependency_manager, output_dir, name, no_input) + do_interactive(location, runtime, dependency_manager, output_dir, name, app_template, no_input) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index c58174f7b1..6cc70be7e8 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -12,17 +12,19 @@ from pathlib import Path # must come after Py2.7 deprecation from samcli.cli.main import global_cfg +from samcli.commands.exceptions import UserException from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING class InitTemplates: - def __init__(self): + def __init__(self, no_interactive=False): # self._repo_url = "https://github.com/awslabs/aws-sam-cli-app-templates.git" # testing only, delete this and uncomment the above line before shipping self._repo_url = "git@github.com:awslabs/aws-sam-cli-app-templates.git" self._repo_name = "aws-sam-cli-app-templates" self.repo_path = None self.clone_attempted = False + self._no_interactive = no_interactive def prompt_for_location(self, runtime, dependency_manager): options = self.init_options(runtime, dependency_manager) @@ -49,6 +51,23 @@ def prompt_for_location(self, runtime, dependency_manager): else: raise UserException("Invalid template. This should not be possible, please raise an issue.") + def location_from_app_template(self, runtime, dependency_manager, app_template): + options = self.init_options(runtime, dependency_manager) + try: + template = next(item for item in options if self._check_app_template(item, app_template)) + if template.get("init_location") is not None: + return template["init_location"] + elif template.get("directory") is not None: + return os.path.join(self.repo_path, template["directory"]) + else: + raise UserException("Invalid template. This should not be possible, please raise an issue.") + except StopIteration: + msg = "Can't find application template " + app_template + " - check valid values in interactive init." + raise UserException(msg) + + def _check_app_template(self, entry, app_template): + return entry["appTemplate"] == app_template + def init_options(self, runtime, dependency_manager): if self.clone_attempted is False: self._clone_repo() @@ -73,6 +92,7 @@ def _init_options_from_bundle(self, runtime, dependency_manager): for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): if runtime in mapping["runtimes"] or any([r.startswith(runtime) for r in mapping["runtimes"]]): if not dependency_manager or dependency_manager == mapping["dependency_manager"]: + mapping["appTemplate"] = "hello-world" # when bundled, use this default template name return [mapping] msg = "Lambda Runtime {} and dependency manager {} does not have an available initialization template.".format( runtime, dependency_manager @@ -97,12 +117,17 @@ def _clone_repo(self): def _should_clone_repo(self, expected_path): path = Path(expected_path) if path.exists(): - overwrite = click.confirm("Init templates exist on disk. Do you wish to update?") - if overwrite: - # possible alternative: pull down first, THEN delete old version - safer if there is a problem - shutil.rmtree(expected_path) # fail hard if there is an issue - return True + if not self._no_interactive: + overwrite = click.confirm("Init templates exist on disk. Do you wish to update?") + if overwrite: + # possible alternative: pull down first, THEN delete old version - safer if there is a problem + shutil.rmtree(expected_path) # fail hard if there is an issue + return True self.repo_path = expected_path return False else: - return True + if self._no_interactive: + return True + else: + do_clone = click.confirm("This process will clone app templates from https://github.com/awslabs/aws-sam-cli-app-templates - is this ok?") + return do_clone diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index 60de6c5bc6..4b890fef76 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -5,14 +5,15 @@ from samcli.commands.init.init_templates import InitTemplates -def do_interactive(location, runtime, dependency_manager, output_dir, name, no_input): +def do_interactive(location, runtime, dependency_manager, output_dir, name, app_template, no_input): extra_context = None if not location: - print( - "1 - Use a Managed Application Template\n2 - Provide a Custom Location" - ) - location_opt_choice = click.prompt("Location Choice", type=click.Choice(['1', '2']), show_choices=False) - if location_opt_choice == '2': + if app_template: + location_opt_choice = "1" + else: + print ("1 - Use a Managed Application Template\n2 - Provide a Custom Location") + location_opt_choice = click.prompt("Location Choice", type=click.Choice(["1", "2"]), show_choices=False) + if location_opt_choice == "2": location = click.prompt("Template location (git, mercurial, http(s), zip, path)", type=str) else: if not name: @@ -29,9 +30,15 @@ def do_interactive(location, runtime, dependency_manager, output_dir, name, no_i "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] ) templates = InitTemplates() - location = templates.prompt_for_location(runtime, dependency_manager) - no_input = True # because we specified the template ourselves - extra_context = {"project_name": name, "runtime": runtime} + if app_template is not None: + # need to get the init templates, and select by name + location = templates.location_from_app_template(runtime, dependency_manager, app_template) + no_input = True + extra_context = {"project_name": name, "runtime": runtime} + else: + location = templates.prompt_for_location(runtime, dependency_manager) + no_input = True # because we specified the template ourselves + extra_context = {"project_name": name, "runtime": runtime} if not output_dir: output_dir = click.prompt("Output Directory", type=click.Path(), default=".") do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) From 80f8307a5cc07ed7d6ad39cf8f5515b151b8ec53 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 7 Oct 2019 14:46:58 -0700 Subject: [PATCH 07/49] WIP: Existing Init Unit Tests Pass More tests needed to meet the coverage bar since there's a lot of new code, but the existing tests work. --- samcli/commands/init/__init__.py | 10 ++++++---- samcli/commands/init/init_templates.py | 16 +++++++++------- tests/unit/commands/init/test_cli.py | 18 ++++++++++++++---- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index edf1375870..d293b6b910 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -56,10 +56,12 @@ @pass_context @track_command def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input): - do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input) # pragma: no cover + do_cli( + ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input + ) # pragma: no cover -def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input): +def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input, auto_clone=True): # check for mutually exclusive parameters if location and app_template: msg = """ @@ -75,12 +77,12 @@ def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_di # need to turn app_template into a location before we generate extra_context = None if app_template: - templates = InitTemplates(no_interactive) + templates = InitTemplates(no_interactive, auto_clone) location = templates.location_from_app_template(runtime, dependency_manager, app_template) no_input = True extra_context = {"project_name": name, "runtime": runtime} if not output_dir: - output_dir = "." # default - should I lift this to overall options and seed default? + output_dir = "." # default - should I lift this to overall options and seed default? do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) elif no_interactive: error_msg = """ diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 6cc70be7e8..353ec4440e 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -17,14 +17,14 @@ class InitTemplates: - def __init__(self, no_interactive=False): - # self._repo_url = "https://github.com/awslabs/aws-sam-cli-app-templates.git" - # testing only, delete this and uncomment the above line before shipping - self._repo_url = "git@github.com:awslabs/aws-sam-cli-app-templates.git" + def __init__(self, no_interactive=False, auto_clone=True): + self._repo_url = "https://github.com/awslabs/aws-sam-cli-app-templates.git" + # self._repo_url = "git@github.com:awslabs/aws-sam-cli-app-templates.git" self._repo_name = "aws-sam-cli-app-templates" self.repo_path = None self.clone_attempted = False self._no_interactive = no_interactive + self._auto_clone = auto_clone def prompt_for_location(self, runtime, dependency_manager): options = self.init_options(runtime, dependency_manager) @@ -92,7 +92,7 @@ def _init_options_from_bundle(self, runtime, dependency_manager): for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): if runtime in mapping["runtimes"] or any([r.startswith(runtime) for r in mapping["runtimes"]]): if not dependency_manager or dependency_manager == mapping["dependency_manager"]: - mapping["appTemplate"] = "hello-world" # when bundled, use this default template name + mapping["appTemplate"] = "hello-world" # when bundled, use this default template name return [mapping] msg = "Lambda Runtime {} and dependency manager {} does not have an available initialization template.".format( runtime, dependency_manager @@ -127,7 +127,9 @@ def _should_clone_repo(self, expected_path): return False else: if self._no_interactive: - return True + return self._auto_clone else: - do_clone = click.confirm("This process will clone app templates from https://github.com/awslabs/aws-sam-cli-app-templates - is this ok?") + do_clone = click.confirm( + "This process will clone app templates from https://github.com/awslabs/aws-sam-cli-app-templates - is this ok?" + ) return do_clone diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index 8d15a2c274..f0f2c59e4e 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch +from mock import patch, ANY from samcli.commands.init import do_cli as init_cli from samcli.local.init.exceptions import GenerateProjectFailedError @@ -9,33 +9,40 @@ class TestCli(TestCase): def setUp(self): self.ctx = None + self.no_interactive = True self.location = None self.runtime = "python3.6" self.dependency_manager = "pip" self.output_dir = "." self.name = "testing project" + self.app_template = "hello-world" self.no_input = False + self.extra_context = {'project_name': 'testing project', 'runtime': 'python3.6'} - @patch("samcli.commands.init.generate_project") + @patch("samcli.commands.init.init_generator.generate_project") def test_init_cli(self, generate_project_patch): # GIVEN generate_project successfully created a project # WHEN a project name has been passed init_cli( ctx=self.ctx, + no_interactive=self.no_interactive, location=self.location, runtime=self.runtime, dependency_manager=self.dependency_manager, output_dir=self.output_dir, name=self.name, + app_template=self.app_template, no_input=self.no_input, + auto_clone=False ) # THEN we should receive no errors generate_project_patch.assert_called_once_with( - self.location, self.runtime, self.dependency_manager, self.output_dir, self.name, self.no_input + # need to change the location validation check + ANY, self.runtime, self.dependency_manager, self.output_dir, self.name, True, self.extra_context ) - @patch("samcli.commands.init.generate_project") + @patch("samcli.commands.init.init_generator.generate_project") def test_init_cli_generate_project_fails(self, generate_project_patch): # GIVEN generate_project fails to create a project @@ -48,12 +55,15 @@ def test_init_cli_generate_project_fails(self, generate_project_patch): with self.assertRaises(UserException): init_cli( self.ctx, + no_interactive=self.no_interactive, location="self.location", runtime=self.runtime, dependency_manager=self.dependency_manager, output_dir=self.output_dir, name=self.name, + app_template=self.app_template, no_input=self.no_input, + auto_clone=False ) generate_project_patch.assert_called_with( From 58a6e4263f0c4e16c55dd196f66cf7ab30578eb4 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 8 Oct 2019 09:30:40 -0700 Subject: [PATCH 08/49] WIP: Clean Up Lint Warnings, Add Go Dep Manager With dependency-manager required, cannot have go showing a None type for that field. --- samcli/commands/init/__init__.py | 13 ++++++- samcli/commands/init/init_templates.py | 34 ++++++++----------- samcli/commands/init/interactive_init_flow.py | 2 +- samcli/local/common/runtime_template.py | 7 ++-- tests/unit/commands/init/test_cli.py | 14 +++++--- 5 files changed, 39 insertions(+), 31 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index d293b6b910..e98e61e7af 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -61,7 +61,18 @@ def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, ) # pragma: no cover -def do_cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, name, app_template, no_input, auto_clone=True): +def do_cli( + ctx, + no_interactive, + location, + runtime, + dependency_manager, + output_dir, + name, + app_template, + no_input, + auto_clone=True, +): # check for mutually exclusive parameters if location and app_template: msg = """ diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 353ec4440e..12788bb512 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -2,7 +2,6 @@ Manages the set of application templates. """ -import click import itertools import json import os @@ -11,6 +10,8 @@ from pathlib import Path # must come after Py2.7 deprecation +import click + from samcli.cli.main import global_cfg from samcli.commands.exceptions import UserException from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING @@ -32,15 +33,11 @@ def prompt_for_location(self, runtime, dependency_manager): choice_num = 1 for o in options: if o.get("displayName") is not None: - print (choice_num, "-", o.get("displayName")) + msg = str(choice_num) + " - " + o.get("displayName") + click.echo(msg) else: - print ( - choice_num, - "- Default Template for runtime", - runtime, - "with dependency manager", - dependency_manager, - ) + msg = str(choice_num) + " - Default Template for runtime " + runtime + " with dependency manager " + dependency_manager + click.echo(msg) choice_num = choice_num + 1 choice = click.prompt("Template Selection", type=click.Choice(choices), show_choices=False) template_md = options[int(choice) - 1] # zero index @@ -73,8 +70,7 @@ def init_options(self, runtime, dependency_manager): self._clone_repo() if self.repo_path is None: return self._init_options_from_bundle(runtime, dependency_manager) - else: - return self._init_options_from_manifest(runtime, dependency_manager) + return self._init_options_from_manifest(runtime, dependency_manager) def _init_options_from_manifest(self, runtime, dependency_manager): manifest_path = os.path.join(self.repo_path, "manifest.json") @@ -85,8 +81,7 @@ def _init_options_from_manifest(self, runtime, dependency_manager): if templates is None: # Fallback to bundled templates return self._init_options_from_bundle(runtime, dependency_manager) - else: - return templates + return templates def _init_options_from_bundle(self, runtime, dependency_manager): for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): @@ -110,9 +105,9 @@ def _clone_repo(self): except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): - print ("WARN: Could not clone app template repo.") + click.echo("WARN: Could not clone app template repo.") # do we ever want to hard fail? - self._clone_attempted = True + self.clone_attempted = True def _should_clone_repo(self, expected_path): path = Path(expected_path) @@ -128,8 +123,7 @@ def _should_clone_repo(self, expected_path): else: if self._no_interactive: return self._auto_clone - else: - do_clone = click.confirm( - "This process will clone app templates from https://github.com/awslabs/aws-sam-cli-app-templates - is this ok?" - ) - return do_clone + do_clone = click.confirm( + "This process will clone app templates from https://github.com/awslabs/aws-sam-cli-app-templates - is this ok?" + ) + return do_clone diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index 4b890fef76..84a4a411fd 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -11,7 +11,7 @@ def do_interactive(location, runtime, dependency_manager, output_dir, name, app_ if app_template: location_opt_choice = "1" else: - print ("1 - Use a Managed Application Template\n2 - Provide a Custom Location") + click.echo("1 - Use a Managed Application Template\n2 - Provide a Custom Location") location_opt_choice = click.prompt("Location Choice", type=click.Choice(["1", "2"]), show_choices=False) if location_opt_choice == "2": location = click.prompt("Template location (git, mercurial, http(s), zip, path)", type=str) diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index da8dd9fd85..35447f9b98 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -57,7 +57,7 @@ "go": [ { "runtimes": ["go1.x"], - "dependency_manager": None, + "dependency_manager": "mod", "init_location": os.path.join(_templates, "cookiecutter-aws-sam-hello-golang"), "build": False, } @@ -89,7 +89,7 @@ "dotnetcore2.1": ["cli-package"], "dotnetcore2.0": ["cli-package"], "dotnetcore1.0": ["cli-package"], - "go1.x": None, + "go1.x": ["mod"], "java8": ["maven", "gradle"], } @@ -107,6 +107,3 @@ ) INIT_RUNTIMES = RUNTIMES.union(RUNTIME_DEP_TEMPLATE_MAPPING.keys()) - -# NOTE(TheSriram): Default Runtime Choice when runtime is not chosen -DEFAULT_RUNTIME = RUNTIME_DEP_TEMPLATE_MAPPING["nodejs"][0]["runtimes"][0] diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index f0f2c59e4e..f0759454b1 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -17,7 +17,7 @@ def setUp(self): self.name = "testing project" self.app_template = "hello-world" self.no_input = False - self.extra_context = {'project_name': 'testing project', 'runtime': 'python3.6'} + self.extra_context = {"project_name": "testing project", "runtime": "python3.6"} @patch("samcli.commands.init.init_generator.generate_project") def test_init_cli(self, generate_project_patch): @@ -33,13 +33,19 @@ def test_init_cli(self, generate_project_patch): name=self.name, app_template=self.app_template, no_input=self.no_input, - auto_clone=False + auto_clone=False, ) # THEN we should receive no errors generate_project_patch.assert_called_once_with( # need to change the location validation check - ANY, self.runtime, self.dependency_manager, self.output_dir, self.name, True, self.extra_context + ANY, + self.runtime, + self.dependency_manager, + self.output_dir, + self.name, + True, + self.extra_context, ) @patch("samcli.commands.init.init_generator.generate_project") @@ -63,7 +69,7 @@ def test_init_cli_generate_project_fails(self, generate_project_patch): name=self.name, app_template=self.app_template, no_input=self.no_input, - auto_clone=False + auto_clone=False, ) generate_project_patch.assert_called_with( From ad09dc35c53767bc38787f7d5bba01deab36276e Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Wed, 9 Oct 2019 11:47:18 -0700 Subject: [PATCH 09/49] Linting Improvements Primarily, refactoring the different primary interactive threads into different methods. --- samcli/commands/init/init_templates.py | 7 +- samcli/commands/init/interactive_init_flow.py | 65 +++++++++++-------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 12788bb512..ce784f372c 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -20,7 +20,6 @@ class InitTemplates: def __init__(self, no_interactive=False, auto_clone=True): self._repo_url = "https://github.com/awslabs/aws-sam-cli-app-templates.git" - # self._repo_url = "git@github.com:awslabs/aws-sam-cli-app-templates.git" self._repo_name = "aws-sam-cli-app-templates" self.repo_path = None self.clone_attempted = False @@ -81,6 +80,9 @@ def _init_options_from_manifest(self, runtime, dependency_manager): if templates is None: # Fallback to bundled templates return self._init_options_from_bundle(runtime, dependency_manager) + if dependency_manager is not None: + templates_by_dep = itertools.takewhile(lambda x: x["dependencyManager"] == dependency_manager, templates) + return list(templates_by_dep) return templates def _init_options_from_bundle(self, runtime, dependency_manager): @@ -100,13 +102,11 @@ def _clone_repo(self): if self._should_clone_repo(expected_path): try: subprocess.check_output(["git", "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT) - # check for repo in path? self.repo_path = expected_path except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): click.echo("WARN: Could not clone app template repo.") - # do we ever want to hard fail? self.clone_attempted = True def _should_clone_repo(self, expected_path): @@ -115,7 +115,6 @@ def _should_clone_repo(self, expected_path): if not self._no_interactive: overwrite = click.confirm("Init templates exist on disk. Do you wish to update?") if overwrite: - # possible alternative: pull down first, THEN delete old version - safer if there is a problem shutil.rmtree(expected_path) # fail hard if there is an issue return True self.repo_path = expected_path diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index 84a4a411fd..e490fa2f8e 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -6,39 +6,50 @@ def do_interactive(location, runtime, dependency_manager, output_dir, name, app_template, no_input): - extra_context = None - if not location: + if location: + if not output_dir: + output_dir = click.prompt("Output Directory", type=click.Path(), default=".") + do_generate(location, runtime, dependency_manager, output_dir, name, no_input, None) + else: if app_template: location_opt_choice = "1" else: click.echo("1 - Use a Managed Application Template\n2 - Provide a Custom Location") location_opt_choice = click.prompt("Location Choice", type=click.Choice(["1", "2"]), show_choices=False) if location_opt_choice == "2": - location = click.prompt("Template location (git, mercurial, http(s), zip, path)", type=str) + _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input) else: - if not name: - name = click.prompt("Project Name", type=str) - if not runtime: - # TODO: Better output than click default choices. - runtime = click.prompt("Runtime", type=click.Choice(RUNTIMES)) - if not dependency_manager: - valid_dep_managers = RUNTIME_TO_DEPENDENCY_MANAGERS.get(runtime) - if valid_dep_managers is None: - dependency_manager = None - else: - dependency_manager = click.prompt( - "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] - ) - templates = InitTemplates() - if app_template is not None: - # need to get the init templates, and select by name - location = templates.location_from_app_template(runtime, dependency_manager, app_template) - no_input = True - extra_context = {"project_name": name, "runtime": runtime} + _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input) + +def _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input): + location = click.prompt("Template location (git, mercurial, http(s), zip, path)", type=str) + if not output_dir: + output_dir = click.prompt("Output Directory", type=click.Path(), default=".") + do_generate(location, runtime, dependency_manager, output_dir, name, no_input, None) + +def _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input): + extra_context = None + if not name: + name = click.prompt("Project Name", type=str) + if not runtime: + runtime = click.prompt("Runtime", type=click.Choice(RUNTIMES)) + if not dependency_manager: + valid_dep_managers = RUNTIME_TO_DEPENDENCY_MANAGERS.get(runtime) + if valid_dep_managers is None: + dependency_manager = None else: - location = templates.prompt_for_location(runtime, dependency_manager) - no_input = True # because we specified the template ourselves - extra_context = {"project_name": name, "runtime": runtime} - if not output_dir: - output_dir = click.prompt("Output Directory", type=click.Path(), default=".") + dependency_manager = click.prompt( + "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] + ) + templates = InitTemplates() + if app_template is not None: + location = templates.location_from_app_template(runtime, dependency_manager, app_template) + no_input = True + extra_context = {"project_name": name, "runtime": runtime} + else: + location = templates.prompt_for_location(runtime, dependency_manager) + no_input = True # because we specified the template ourselves + extra_context = {"project_name": name, "runtime": runtime} + if not output_dir: + output_dir = click.prompt("Output Directory", type=click.Path(), default=".") do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) From 535544b68672a110aeac495de379192d5cd2eb87 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Wed, 9 Oct 2019 12:22:18 -0700 Subject: [PATCH 10/49] Add Docstrings, Fix Branch Indentation --- samcli/commands/init/init_generator.py | 3 ++ samcli/commands/init/interactive_init_flow.py | 39 ++++++++++--------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/samcli/commands/init/init_generator.py b/samcli/commands/init/init_generator.py index cf49e8683d..5c1111f3fc 100644 --- a/samcli/commands/init/init_generator.py +++ b/samcli/commands/init/init_generator.py @@ -1,3 +1,6 @@ +""" +Cookiecutter-based generation logic for project templates. +""" import click from samcli.commands.exceptions import UserException diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index e490fa2f8e..d4e90c5b87 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -1,3 +1,6 @@ +""" +Isolates interactive init prompt flow. Expected to call generator logic at end of flow. +""" import click from samcli.local.common.runtime_template import RUNTIMES, RUNTIME_TO_DEPENDENCY_MANAGERS @@ -31,25 +34,25 @@ def _generate_from_app_template(location, runtime, dependency_manager, output_di extra_context = None if not name: name = click.prompt("Project Name", type=str) - if not runtime: - runtime = click.prompt("Runtime", type=click.Choice(RUNTIMES)) - if not dependency_manager: - valid_dep_managers = RUNTIME_TO_DEPENDENCY_MANAGERS.get(runtime) - if valid_dep_managers is None: - dependency_manager = None - else: - dependency_manager = click.prompt( - "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] - ) - templates = InitTemplates() - if app_template is not None: - location = templates.location_from_app_template(runtime, dependency_manager, app_template) - no_input = True - extra_context = {"project_name": name, "runtime": runtime} + if not runtime: + runtime = click.prompt("Runtime", type=click.Choice(RUNTIMES)) + if not dependency_manager: + valid_dep_managers = RUNTIME_TO_DEPENDENCY_MANAGERS.get(runtime) + if valid_dep_managers is None: + dependency_manager = None else: - location = templates.prompt_for_location(runtime, dependency_manager) - no_input = True # because we specified the template ourselves - extra_context = {"project_name": name, "runtime": runtime} + dependency_manager = click.prompt( + "Dependency Manager", type=click.Choice(valid_dep_managers), default=valid_dep_managers[0] + ) + templates = InitTemplates() + if app_template is not None: + location = templates.location_from_app_template(runtime, dependency_manager, app_template) + no_input = True + extra_context = {"project_name": name, "runtime": runtime} + else: + location = templates.prompt_for_location(runtime, dependency_manager) + no_input = True # because we specified the template ourselves + extra_context = {"project_name": name, "runtime": runtime} if not output_dir: output_dir = click.prompt("Output Directory", type=click.Path(), default=".") do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) From b6d7cf692151c9233c9dd4d2805ecd7c43f7b3f0 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Wed, 9 Oct 2019 16:10:17 -0700 Subject: [PATCH 11/49] Remove Planning Comment --- samcli/commands/init/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index e98e61e7af..f3e4aedc38 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -93,7 +93,7 @@ def do_cli( no_input = True extra_context = {"project_name": name, "runtime": runtime} if not output_dir: - output_dir = "." # default - should I lift this to overall options and seed default? + output_dir = "." do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) elif no_interactive: error_msg = """ From 64b45ead96811f3897d75f7a06fa85692c500fd1 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Thu, 10 Oct 2019 09:55:08 -0700 Subject: [PATCH 12/49] Check for Multiple Git Executables Windows compatibility check, essentially, and this commit also includes a Black reformatting. --- samcli/commands/init/init_templates.py | 31 +++++++++++++++++-- samcli/commands/init/interactive_init_flow.py | 2 ++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index ce784f372c..720e43b52c 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -5,6 +5,7 @@ import itertools import json import os +import platform import shutil import subprocess @@ -35,7 +36,13 @@ def prompt_for_location(self, runtime, dependency_manager): msg = str(choice_num) + " - " + o.get("displayName") click.echo(msg) else: - msg = str(choice_num) + " - Default Template for runtime " + runtime + " with dependency manager " + dependency_manager + msg = ( + str(choice_num) + + " - Default Template for runtime " + + runtime + + " with dependency manager " + + dependency_manager + ) click.echo(msg) choice_num = choice_num + 1 choice = click.prompt("Template Selection", type=click.Choice(choices), show_choices=False) @@ -81,7 +88,9 @@ def _init_options_from_manifest(self, runtime, dependency_manager): # Fallback to bundled templates return self._init_options_from_bundle(runtime, dependency_manager) if dependency_manager is not None: - templates_by_dep = itertools.takewhile(lambda x: x["dependencyManager"] == dependency_manager, templates) + templates_by_dep = itertools.takewhile( + lambda x: x["dependencyManager"] == dependency_manager, templates + ) return list(templates_by_dep) return templates @@ -101,14 +110,30 @@ def _clone_repo(self): expected_path = os.path.normpath(os.path.join(shared_dir, self._repo_name)) if self._should_clone_repo(expected_path): try: - subprocess.check_output(["git", "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT) + subprocess.check_output( + [self._git_executable, "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT + ) self.repo_path = expected_path + except OSError as os_error: + output = os_error.output.decode("utf-8") + click.echo("WARN: Can't clone app repo, git executable not found.") except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): click.echo("WARN: Could not clone app template repo.") self.clone_attempted = True + def _git_executable(): + execname = "git" # do we want to abstract the AWS CLI version of this? + if platform.system().lower() == "windows": + options = ["{}.cmd".format(execname), "{}.exe".format(execname), execname] + else: + options = [execname] + for name in options: + subprocess.Popen([name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # No exception. Let's pick this + return name + def _should_clone_repo(self, expected_path): path = Path(expected_path) if path.exists(): diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index d4e90c5b87..3398d9319d 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -24,12 +24,14 @@ def do_interactive(location, runtime, dependency_manager, output_dir, name, app_ else: _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input) + def _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input): location = click.prompt("Template location (git, mercurial, http(s), zip, path)", type=str) if not output_dir: output_dir = click.prompt("Output Directory", type=click.Path(), default=".") do_generate(location, runtime, dependency_manager, output_dir, name, no_input, None) + def _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input): extra_context = None if not name: From 89930871c4b0ffe39bcddafa257fc8b9ac2766b6 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Thu, 10 Oct 2019 10:39:07 -0700 Subject: [PATCH 13/49] Add Click Command Unit Tests for Init This enables us to properly test prompts and improves code coverage. We also managed to entirely remove a code path (location cannot currently be missing when going to interactive). Remaining work is to improve unit test coverage of init templates. --- samcli/commands/init/interactive_init_flow.py | 21 +-- tests/unit/commands/init/test_cli.py | 138 +++++++++++++++++- 2 files changed, 144 insertions(+), 15 deletions(-) diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index 3398d9319d..dd153079b4 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -9,20 +9,15 @@ def do_interactive(location, runtime, dependency_manager, output_dir, name, app_template, no_input): - if location: - if not output_dir: - output_dir = click.prompt("Output Directory", type=click.Path(), default=".") - do_generate(location, runtime, dependency_manager, output_dir, name, no_input, None) + if app_template: + location_opt_choice = "1" else: - if app_template: - location_opt_choice = "1" - else: - click.echo("1 - Use a Managed Application Template\n2 - Provide a Custom Location") - location_opt_choice = click.prompt("Location Choice", type=click.Choice(["1", "2"]), show_choices=False) - if location_opt_choice == "2": - _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input) - else: - _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input) + click.echo("1 - Use a Managed Application Template\n2 - Provide a Custom Location") + location_opt_choice = click.prompt("Location Choice", type=click.Choice(["1", "2"]), show_choices=False) + if location_opt_choice == "2": + _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input) + else: + _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input) def _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input): diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index f0759454b1..10600a4b90 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -1,6 +1,10 @@ from unittest import TestCase from mock import patch, ANY +import click +from click.testing import CliRunner + +from samcli.commands.init import cli as init_cmd from samcli.commands.init import do_cli as init_cli from samcli.local.init.exceptions import GenerateProjectFailedError from samcli.commands.exceptions import UserException @@ -29,7 +33,7 @@ def test_init_cli(self, generate_project_patch): location=self.location, runtime=self.runtime, dependency_manager=self.dependency_manager, - output_dir=self.output_dir, + output_dir=None, name=self.name, app_template=self.app_template, no_input=self.no_input, @@ -48,6 +52,136 @@ def test_init_cli(self, generate_project_patch): self.extra_context, ) + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_interactive(self, generate_project_patch): + # WHEN the user follows interactice init prompts + + # 1: selecting managed templates + # test-project: response to name + # ruby2.5: response to runtime + # bundler: response to dependency manager + # N: Don't clone/update the source repo + # 1: First choice will always be the hello world example + user_input = """ +1 +test-project +ruby2.5 +bundler +N +1 +. + """ + runner = CliRunner() + result = runner.invoke(init_cmd, input=user_input) + + # THEN we should receive no errors + assert not result.exception + generate_project_patch.assert_called_once_with( + # need to change the location validation check + ANY, + "ruby2.5", + "bundler", + ".", + "test-project", + True, + {"project_name": "test-project", "runtime": "ruby2.5"}, + ) + + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_int_with_app_template(self, generate_project_patch): + # WHEN the user follows interactice init prompts + + # test-project: response to name + # ruby2.5: response to runtime + # bundler: response to dependency manager + # N: Don't clone/update the source repo + # .: output dir + user_input = """ +test-project +ruby2.5 +bundler +N +. + """ + runner = CliRunner() + result = runner.invoke(init_cmd, ["--app-template", "hello-world"], input=user_input) + + # THEN we should receive no errors + assert not result.exception + generate_project_patch.assert_called_once_with( + # need to change the location validation check + ANY, + "ruby2.5", + "bundler", + ".", + "test-project", + True, + {"project_name": "test-project", "runtime": "ruby2.5"}, + ) + + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_int_from_location(self, generate_project_patch): + # WHEN the user follows interactice init prompts + + # 2: selecting custom location + # foo: the "location" + # output/: the "output dir" + user_input = """ +2 +foo +output/ + """ + + runner = CliRunner() + result = runner.invoke(init_cmd, input=user_input) + + # THEN we should receive no errors + assert not result.exception + generate_project_patch.assert_called_once_with( + # need to change the location validation check + "foo", + None, + None, + "output/", + None, + False, + None, + ) + + def test_init_cli_missing_params_fails(self): + # WHEN we call init without necessary parameters + # THEN we should receive a UserException + with self.assertRaises(UserException): + init_cli( + self.ctx, + no_interactive=True, + location=None, + runtime=None, + dependency_manager=None, + output_dir=None, + name=None, + app_template=None, + no_input=True, + auto_clone=False, + ) + + def test_init_cli_mutually_exclusive_params_fails(self): + # WHEN we call init without necessary parameters + # THEN we should receive a UserException + with self.assertRaises(UserException): + init_cli( + self.ctx, + no_interactive=self.no_interactive, + location="whatever", + runtime=self.runtime, + dependency_manager=self.dependency_manager, + output_dir=self.output_dir, + name=self.name, + app_template="fails-anyways", + no_input=self.no_input, + auto_clone=False, + ) + @patch("samcli.commands.init.init_generator.generate_project") def test_init_cli_generate_project_fails(self, generate_project_patch): @@ -67,7 +201,7 @@ def test_init_cli_generate_project_fails(self, generate_project_patch): dependency_manager=self.dependency_manager, output_dir=self.output_dir, name=self.name, - app_template=self.app_template, + app_template=None, no_input=self.no_input, auto_clone=False, ) From bf2f2b33e359983f0918c3e55eef4345ed80e709 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Thu, 10 Oct 2019 10:43:22 -0700 Subject: [PATCH 14/49] Fix _git_executable Function --- samcli/commands/init/init_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 720e43b52c..eb916baa4b 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -123,7 +123,7 @@ def _clone_repo(self): click.echo("WARN: Could not clone app template repo.") self.clone_attempted = True - def _git_executable(): + def _git_executable(self): execname = "git" # do we want to abstract the AWS CLI version of this? if platform.system().lower() == "windows": options = ["{}.cmd".format(execname), "{}.exe".format(execname), execname] From 8c53066f2fa8af700b0c6614b663bac9914c87d5 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Thu, 10 Oct 2019 12:03:39 -0700 Subject: [PATCH 15/49] Fix Git Templates Function Call --- samcli/commands/init/init_templates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index eb916baa4b..a8732a3a26 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -111,7 +111,7 @@ def _clone_repo(self): if self._should_clone_repo(expected_path): try: subprocess.check_output( - [self._git_executable, "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT + [self._git_executable(), "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT ) self.repo_path = expected_path except OSError as os_error: @@ -124,7 +124,7 @@ def _clone_repo(self): self.clone_attempted = True def _git_executable(self): - execname = "git" # do we want to abstract the AWS CLI version of this? + execname = "git" if platform.system().lower() == "windows": options = ["{}.cmd".format(execname), "{}.exe".format(execname), execname] else: From 8066ae851d74b0c9cdc64fa055cc5546a41d57bc Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 15 Oct 2019 09:55:58 -0700 Subject: [PATCH 16/49] Use Filter for Template Selection It was incorrect to use 'takewhile', as it terminates as soon as there is a failed check. For Java Maven this caused no templates to be usable. --- samcli/commands/init/init_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index a8732a3a26..1e8251977a 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -88,7 +88,7 @@ def _init_options_from_manifest(self, runtime, dependency_manager): # Fallback to bundled templates return self._init_options_from_bundle(runtime, dependency_manager) if dependency_manager is not None: - templates_by_dep = itertools.takewhile( + templates_by_dep = filter( lambda x: x["dependencyManager"] == dependency_manager, templates ) return list(templates_by_dep) From b09929fd18b3ce155e125554bd808bdc07c027f9 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 15 Oct 2019 10:24:13 -0700 Subject: [PATCH 17/49] Unit Test the Template Class --- samcli/commands/init/init_templates.py | 5 +-- tests/unit/commands/init/test_templates.py | 47 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 tests/unit/commands/init/test_templates.py diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 1e8251977a..4b99ea78ba 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -88,9 +88,7 @@ def _init_options_from_manifest(self, runtime, dependency_manager): # Fallback to bundled templates return self._init_options_from_bundle(runtime, dependency_manager) if dependency_manager is not None: - templates_by_dep = filter( - lambda x: x["dependencyManager"] == dependency_manager, templates - ) + templates_by_dep = filter(lambda x: x["dependencyManager"] == dependency_manager, templates) return list(templates_by_dep) return templates @@ -115,7 +113,6 @@ def _clone_repo(self): ) self.repo_path = expected_path except OSError as os_error: - output = os_error.output.decode("utf-8") click.echo("WARN: Can't clone app repo, git executable not found.") except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py new file mode 100644 index 0000000000..9f8ffdad1b --- /dev/null +++ b/tests/unit/commands/init/test_templates.py @@ -0,0 +1,47 @@ +import json +import click + +from mock import mock_open, patch, PropertyMock, MagicMock +from re import search +from unittest import TestCase + +from samcli.commands.init.init_templates import InitTemplates + + +class TestTemplates(TestCase): + @patch("subprocess.check_output") + @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") + def test_location_from_app_template(self, subprocess_mock, git_exec_mock): + git_exec_mock.return_value = "git" + it = InitTemplates(True) + + manifest = { + "ruby2.5": [ + { + "directory": "mock-ruby-template", + "displayName": "Hello World Example", + "dependencyManager": "bundler", + "appTemplate": "hello-world", + } + ] + } + manifest_json = json.dumps(manifest) + + m = mock_open(read_data=manifest_json) + with patch("samcli.cli.global_config.GlobalConfig.config_dir", new_callable=PropertyMock) as mock_cfg: + mock_cfg.return_value = "/tmp/test-sam" + with patch("samcli.commands.init.init_templates.open", m): + location = it.location_from_app_template("ruby2.5", "bundler", "hello-world") + assert location == "/tmp/test-sam/aws-sam-cli-app-templates/mock-ruby-template" + + @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") + @patch("click.prompt") + def test_fallback_options(self, git_exec_mock, prompt_mock): + prompt_mock.return_value = "1" + with patch("subprocess.check_output", new_callable=MagicMock) as mock_sub: + with patch("samcli.cli.global_config.GlobalConfig.config_dir", new_callable=PropertyMock) as mock_cfg: + mock_sub.side_effect = OSError("Fail") + mock_cfg.return_value = "/tmp/test-sam" + it = InitTemplates(True) + location = it.prompt_for_location("ruby2.5", "bundler") + assert search("cookiecutter-aws-sam-hello-ruby", location) From 694a0be68db81a2c675c473b22fc95923637ccca Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 15 Oct 2019 13:47:40 -0700 Subject: [PATCH 18/49] Additional Test Coverage Now over the 95% threshold, so `make pr` is passing. --- samcli/commands/init/__init__.py | 4 +-- tests/unit/commands/init/test_cli.py | 40 ++++++++++++++++++++-- tests/unit/commands/init/test_templates.py | 14 +++++++- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index f3e4aedc38..2625664716 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -28,7 +28,7 @@ "--no-interactive", is_flag=True, default=False, - help="Disable interactive prompting for values, and fail if any required values are missing.", + help="Disable interactive prompting for init parameters, and fail if any required values are missing.", ) @click.option("-l", "--location", help="Template location (git, mercurial, http(s), zip, path)") @click.option("-r", "--runtime", type=click.Choice(RUNTIMES), help="Lambda Runtime of your app") @@ -50,7 +50,7 @@ "--no-input", is_flag=True, default=False, - help="Disable prompting and accept default values defined template config", + help="Disable Cookiecutter prompting and accept default values defined template config", ) @common_options @pass_context diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index 10600a4b90..e3bfeb53a0 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -52,9 +52,43 @@ def test_init_cli(self, generate_project_patch): self.extra_context, ) + def test_init_fails_invalid_template(self): + # WHEN an unknown app template is passed in + # THEN an exception should be raised + with self.assertRaises(UserException): + init_cli( + ctx=self.ctx, + no_interactive=self.no_interactive, + location=self.location, + runtime=self.runtime, + dependency_manager=self.dependency_manager, + output_dir=None, + name=self.name, + app_template="wrong-and-bad", + no_input=self.no_input, + auto_clone=False, + ) + + def test_init_fails_invalid_dep_mgr(self): + # WHEN an unknown app template is passed in + # THEN an exception should be raised + with self.assertRaises(UserException): + init_cli( + ctx=self.ctx, + no_interactive=self.no_interactive, + location=self.location, + runtime=self.runtime, + dependency_manager="bad-wrong", + output_dir=None, + name=self.name, + app_template=self.app_template, + no_input=self.no_input, + auto_clone=False, + ) + @patch("samcli.commands.init.init_generator.generate_project") def test_init_cli_interactive(self, generate_project_patch): - # WHEN the user follows interactice init prompts + # WHEN the user follows interactive init prompts # 1: selecting managed templates # test-project: response to name @@ -89,7 +123,7 @@ def test_init_cli_interactive(self, generate_project_patch): @patch("samcli.commands.init.init_generator.generate_project") def test_init_cli_int_with_app_template(self, generate_project_patch): - # WHEN the user follows interactice init prompts + # WHEN the user follows interactive init prompts # test-project: response to name # ruby2.5: response to runtime @@ -121,7 +155,7 @@ def test_init_cli_int_with_app_template(self, generate_project_patch): @patch("samcli.commands.init.init_generator.generate_project") def test_init_cli_int_from_location(self, generate_project_patch): - # WHEN the user follows interactice init prompts + # WHEN the user follows interactive init prompts # 2: selecting custom location # foo: the "location" diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index 9f8ffdad1b..54c15e3ed4 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -1,4 +1,5 @@ import json +import subprocess import click from mock import mock_open, patch, PropertyMock, MagicMock @@ -12,7 +13,6 @@ class TestTemplates(TestCase): @patch("subprocess.check_output") @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") def test_location_from_app_template(self, subprocess_mock, git_exec_mock): - git_exec_mock.return_value = "git" it = InitTemplates(True) manifest = { @@ -45,3 +45,15 @@ def test_fallback_options(self, git_exec_mock, prompt_mock): it = InitTemplates(True) location = it.prompt_for_location("ruby2.5", "bundler") assert search("cookiecutter-aws-sam-hello-ruby", location) + + @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") + @patch("click.prompt") + def test_fallback_process_error(self, git_exec_mock, prompt_mock): + prompt_mock.return_value = "1" + with patch("subprocess.check_output", new_callable=MagicMock) as mock_sub: + with patch("samcli.cli.global_config.GlobalConfig.config_dir", new_callable=PropertyMock) as mock_cfg: + mock_sub.side_effect = subprocess.CalledProcessError("fail", "fail", "not found".encode("utf-8")) + mock_cfg.return_value = "/tmp/test-sam" + it = InitTemplates(True) + location = it.prompt_for_location("ruby2.5", "bundler") + assert search("cookiecutter-aws-sam-hello-ruby", location) From 9eb8167b2a1ddd707e0c7cb67fcc77bceaa39bb2 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 15 Oct 2019 13:48:32 -0700 Subject: [PATCH 19/49] Remove Unused Variable --- samcli/commands/init/init_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 4b99ea78ba..f46de4cc6b 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -112,7 +112,7 @@ def _clone_repo(self): [self._git_executable(), "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT ) self.repo_path = expected_path - except OSError as os_error: + except OSError: click.echo("WARN: Can't clone app repo, git executable not found.") except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") From 0378bccda2c8274924c1158d7019e1fab053cf52 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 15 Oct 2019 15:45:57 -0700 Subject: [PATCH 20/49] Address Init PR Comments --- samcli/commands/init/interactive_init_flow.py | 7 +++---- tests/unit/commands/init/test_cli.py | 6 +++--- tests/unit/commands/init/test_templates.py | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index dd153079b4..dbb55a7b44 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -17,7 +17,7 @@ def do_interactive(location, runtime, dependency_manager, output_dir, name, app_ if location_opt_choice == "2": _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input) else: - _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input) + _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template) def _generate_from_location(location, runtime, dependency_manager, output_dir, name, app_template, no_input): @@ -27,7 +27,7 @@ def _generate_from_location(location, runtime, dependency_manager, output_dir, n do_generate(location, runtime, dependency_manager, output_dir, name, no_input, None) -def _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template, no_input): +def _generate_from_app_template(location, runtime, dependency_manager, output_dir, name, app_template): extra_context = None if not name: name = click.prompt("Project Name", type=str) @@ -44,12 +44,11 @@ def _generate_from_app_template(location, runtime, dependency_manager, output_di templates = InitTemplates() if app_template is not None: location = templates.location_from_app_template(runtime, dependency_manager, app_template) - no_input = True extra_context = {"project_name": name, "runtime": runtime} else: location = templates.prompt_for_location(runtime, dependency_manager) - no_input = True # because we specified the template ourselves extra_context = {"project_name": name, "runtime": runtime} + no_input = True if not output_dir: output_dir = click.prompt("Output Directory", type=click.Path(), default=".") do_generate(location, runtime, dependency_manager, output_dir, name, no_input, extra_context) diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index e3bfeb53a0..451189c936 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -109,7 +109,7 @@ def test_init_cli_interactive(self, generate_project_patch): result = runner.invoke(init_cmd, input=user_input) # THEN we should receive no errors - assert not result.exception + self.assertFalse(result.exception) generate_project_patch.assert_called_once_with( # need to change the location validation check ANY, @@ -141,7 +141,7 @@ def test_init_cli_int_with_app_template(self, generate_project_patch): result = runner.invoke(init_cmd, ["--app-template", "hello-world"], input=user_input) # THEN we should receive no errors - assert not result.exception + self.assertFalse(result.exception) generate_project_patch.assert_called_once_with( # need to change the location validation check ANY, @@ -170,7 +170,7 @@ def test_init_cli_int_from_location(self, generate_project_patch): result = runner.invoke(init_cmd, input=user_input) # THEN we should receive no errors - assert not result.exception + self.assertFalse(result.exception) generate_project_patch.assert_called_once_with( # need to change the location validation check "foo", diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index 54c15e3ed4..ceb4ce376c 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -32,7 +32,7 @@ def test_location_from_app_template(self, subprocess_mock, git_exec_mock): mock_cfg.return_value = "/tmp/test-sam" with patch("samcli.commands.init.init_templates.open", m): location = it.location_from_app_template("ruby2.5", "bundler", "hello-world") - assert location == "/tmp/test-sam/aws-sam-cli-app-templates/mock-ruby-template" + self.assertEqual(location, "/tmp/test-sam/aws-sam-cli-app-templates/mock-ruby-template") @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") @patch("click.prompt") @@ -44,7 +44,7 @@ def test_fallback_options(self, git_exec_mock, prompt_mock): mock_cfg.return_value = "/tmp/test-sam" it = InitTemplates(True) location = it.prompt_for_location("ruby2.5", "bundler") - assert search("cookiecutter-aws-sam-hello-ruby", location) + self.assertTrue(search("cookiecutter-aws-sam-hello-ruby", location)) @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") @patch("click.prompt") @@ -56,4 +56,4 @@ def test_fallback_process_error(self, git_exec_mock, prompt_mock): mock_cfg.return_value = "/tmp/test-sam" it = InitTemplates(True) location = it.prompt_for_location("ruby2.5", "bundler") - assert search("cookiecutter-aws-sam-hello-ruby", location) + self.assertTrue(search("cookiecutter-aws-sam-hello-ruby", location)) From 7438d9f2292dfaeb98d1cc6c538309e3a563c112 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Thu, 17 Oct 2019 10:30:22 -0700 Subject: [PATCH 21/49] Update Init Integ Tests --- tests/integration/init/test_init_command.py | 58 ++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/tests/integration/init/test_init_command.py b/tests/integration/init/test_init_command.py index 79acb2527c..95d3719a66 100644 --- a/tests/integration/init/test_init_command.py +++ b/tests/integration/init/test_init_command.py @@ -8,12 +8,68 @@ class TestBasicInitCommand(TestCase): def test_init_command_passes_and_dir_created(self): with tempfile.TemporaryDirectory() as temp: - process = Popen([TestBasicInitCommand._get_command(), "init", "-o", temp]) + process = Popen([ + TestBasicInitCommand._get_command(), "init", + "--runtime", "nodejs10.x", + "--dependency-manager", "npm", + "--app-template", "hello-world", + "--name", "sam-app", + "--no-interactive", + "-o", temp + ]) return_code = process.wait() self.assertEqual(return_code, 0) self.assertTrue(os.path.isdir(temp + "/sam-app")) + def test_init_new_app_template(self): + with tempfile.TemporaryDirectory() as temp: + process = Popen([ + TestBasicInitCommand._get_command(), "init", + "--runtime", "nodejs10.x", + "--dependency-manager", "npm", + "--app-template", "quick-start-from-scratch", + "--name", "qs-scratch", + "--no-interactive", + "-o", temp + ]) + return_code = process.wait() + + self.assertEqual(return_code, 0) + self.assertTrue(os.path.isdir(temp + "/qs-scratch")) + + def test_init_command_java_maven(self): + with tempfile.TemporaryDirectory() as temp: + process = Popen([ + TestBasicInitCommand._get_command(), "init", + "--runtime", "java8", + "--dependency-manager", "maven", + "--app-template", "hello-world", + "--name", "sam-app-maven", + "--no-interactive", + "-o", temp + ]) + return_code = process.wait() + + self.assertEqual(return_code, 0) + self.assertTrue(os.path.isdir(temp + "/sam-app-maven")) + + def test_init_command_java_gradle(self): + with tempfile.TemporaryDirectory() as temp: + process = Popen([ + TestBasicInitCommand._get_command(), "init", + "--runtime", "java8", + "--dependency-manager", "gradle", + "--app-template", "hello-world", + "--name", "sam-app-gradle", + "--no-interactive", + "-o", temp + ]) + return_code = process.wait() + + self.assertEqual(return_code, 0) + self.assertTrue(os.path.isdir(temp + "/sam-app-gradle")) + @staticmethod def _get_command(): command = "sam" From 290032447d467d8e12a7daf6cde603bda4b212db Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Wed, 23 Oct 2019 09:31:28 -0700 Subject: [PATCH 22/49] Run of Black Reformatting --- tests/integration/init/test_init_command.py | 104 +++++++++++++------- 1 file changed, 68 insertions(+), 36 deletions(-) diff --git a/tests/integration/init/test_init_command.py b/tests/integration/init/test_init_command.py index 95d3719a66..6e2e791246 100644 --- a/tests/integration/init/test_init_command.py +++ b/tests/integration/init/test_init_command.py @@ -8,15 +8,23 @@ class TestBasicInitCommand(TestCase): def test_init_command_passes_and_dir_created(self): with tempfile.TemporaryDirectory() as temp: - process = Popen([ - TestBasicInitCommand._get_command(), "init", - "--runtime", "nodejs10.x", - "--dependency-manager", "npm", - "--app-template", "hello-world", - "--name", "sam-app", - "--no-interactive", - "-o", temp - ]) + process = Popen( + [ + TestBasicInitCommand._get_command(), + "init", + "--runtime", + "nodejs10.x", + "--dependency-manager", + "npm", + "--app-template", + "hello-world", + "--name", + "sam-app", + "--no-interactive", + "-o", + temp, + ] + ) return_code = process.wait() self.assertEqual(return_code, 0) @@ -24,15 +32,23 @@ def test_init_command_passes_and_dir_created(self): def test_init_new_app_template(self): with tempfile.TemporaryDirectory() as temp: - process = Popen([ - TestBasicInitCommand._get_command(), "init", - "--runtime", "nodejs10.x", - "--dependency-manager", "npm", - "--app-template", "quick-start-from-scratch", - "--name", "qs-scratch", - "--no-interactive", - "-o", temp - ]) + process = Popen( + [ + TestBasicInitCommand._get_command(), + "init", + "--runtime", + "nodejs10.x", + "--dependency-manager", + "npm", + "--app-template", + "quick-start-from-scratch", + "--name", + "qs-scratch", + "--no-interactive", + "-o", + temp, + ] + ) return_code = process.wait() self.assertEqual(return_code, 0) @@ -40,15 +56,23 @@ def test_init_new_app_template(self): def test_init_command_java_maven(self): with tempfile.TemporaryDirectory() as temp: - process = Popen([ - TestBasicInitCommand._get_command(), "init", - "--runtime", "java8", - "--dependency-manager", "maven", - "--app-template", "hello-world", - "--name", "sam-app-maven", - "--no-interactive", - "-o", temp - ]) + process = Popen( + [ + TestBasicInitCommand._get_command(), + "init", + "--runtime", + "java8", + "--dependency-manager", + "maven", + "--app-template", + "hello-world", + "--name", + "sam-app-maven", + "--no-interactive", + "-o", + temp, + ] + ) return_code = process.wait() self.assertEqual(return_code, 0) @@ -56,15 +80,23 @@ def test_init_command_java_maven(self): def test_init_command_java_gradle(self): with tempfile.TemporaryDirectory() as temp: - process = Popen([ - TestBasicInitCommand._get_command(), "init", - "--runtime", "java8", - "--dependency-manager", "gradle", - "--app-template", "hello-world", - "--name", "sam-app-gradle", - "--no-interactive", - "-o", temp - ]) + process = Popen( + [ + TestBasicInitCommand._get_command(), + "init", + "--runtime", + "java8", + "--dependency-manager", + "gradle", + "--app-template", + "hello-world", + "--name", + "sam-app-gradle", + "--no-interactive", + "-o", + temp, + ] + ) return_code = process.wait() self.assertEqual(return_code, 0) From ee91629e289eaa634809e6d691ce701426bab318 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Wed, 23 Oct 2019 17:13:09 -0500 Subject: [PATCH 23/49] Deprecate: Remove Python2.7 installation support (#1416) --- appveyor-windows.yml | 1 + appveyor.yml | 6 +++ pyproject.toml | 3 +- pytest.ini | 4 +- requirements/base.txt | 3 -- requirements/dev.txt | 25 ++++------- samcli/cli/command.py | 19 ++------ samcli/cli/context.py | 6 ++- samcli/cli/global_config.py | 7 +-- samcli/commands/_utils/template.py | 9 +--- samcli/commands/build/build_context.py | 10 ++--- samcli/commands/build/exceptions.py | 1 - samcli/commands/init/__init__.py | 2 +- .../local/cli_common/invoke_context.py | 18 +++----- samcli/commands/local/cli_common/options.py | 7 +-- .../local/cli_common/user_exceptions.py | 18 -------- samcli/commands/local/generate_event/cli.py | 1 - samcli/commands/local/lib/api_collector.py | 2 +- samcli/commands/local/lib/api_provider.py | 3 +- samcli/commands/local/lib/cfn_api_provider.py | 2 + .../local/lib/cfn_base_api_provider.py | 2 +- samcli/commands/local/lib/debug_context.py | 2 +- samcli/commands/local/lib/exceptions.py | 4 -- .../lib/generated_sample_events/events.py | 2 +- .../commands/local/lib/local_api_service.py | 4 +- samcli/commands/local/lib/local_lambda.py | 4 +- .../local/lib/local_lambda_service.py | 2 +- samcli/commands/local/lib/provider.py | 12 +++--- .../commands/local/lib/sam_base_provider.py | 2 +- .../local/lib/sam_function_provider.py | 5 +-- .../local/lib/swagger/integration_uri.py | 5 ++- samcli/commands/local/lib/swagger/parser.py | 4 +- samcli/commands/local/lib/swagger/reader.py | 14 +++--- samcli/commands/local/local.py | 1 - samcli/commands/logs/logs_context.py | 4 +- samcli/commands/publish/command.py | 2 +- samcli/commands/validate/lib/exceptions.py | 2 - .../validate/lib/sam_template_validator.py | 8 ++-- samcli/commands/validate/validate.py | 2 +- samcli/lib/build/app_builder.py | 19 +++----- samcli/lib/build/workflow_config.py | 2 +- .../intrinsic_property_resolver.py | 5 ++- .../intrinsics_symbol_table.py | 4 +- samcli/lib/logs/event.py | 2 +- samcli/lib/logs/fetcher.py | 2 +- samcli/lib/logs/formatter.py | 8 ++-- samcli/lib/logs/provider.py | 2 +- .../samlib/resource_metadata_normalizer.py | 2 +- samcli/lib/samlib/wrapper.py | 4 +- samcli/lib/telemetry/telemetry.py | 4 +- samcli/lib/utils/colors.py | 2 +- samcli/lib/utils/osutils.py | 22 +--------- samcli/lib/utils/sam_logging.py | 2 +- samcli/lib/utils/stream_writer.py | 2 +- samcli/local/apigw/local_apigw_service.py | 4 +- samcli/local/apigw/path_converter.py | 2 +- samcli/local/apigw/service_error_responses.py | 2 +- samcli/local/common/runtime_template.py | 19 +++----- samcli/local/docker/container.py | 2 +- samcli/local/docker/lambda_build_container.py | 6 +-- .../local/docker/lambda_debug_entrypoint.py | 2 +- samcli/local/docker/lambda_image.py | 10 ++--- samcli/local/docker/manager.py | 8 ++-- samcli/local/docker/utils.py | 6 +-- samcli/local/events/api_event.py | 6 +-- .../lambda_service/lambda_error_responses.py | 2 +- .../local_lambda_invoke_service.py | 2 + samcli/local/lambdafn/config.py | 2 +- samcli/local/lambdafn/env_vars.py | 2 +- samcli/local/lambdafn/exceptions.py | 2 - samcli/local/lambdafn/runtime.py | 15 ++++--- samcli/local/lambdafn/zip.py | 8 +--- samcli/local/layers/layer_downloader.py | 11 ++--- samcli/local/services/base_local_service.py | 4 +- samcli/yamlhelper.py | 4 +- scripts/check-isolated-needs-update.py | 2 +- scripts/check-requirements.py | 12 +++--- setup.py | 5 +-- .../commands/cli/test_global_config.py | 8 +--- .../local/lib/test_local_api_service.py | 2 +- .../commands/local/lib/test_local_lambda.py | 2 +- .../lib/test_sam_template_validator.py | 2 +- .../local/apigw/test_local_apigw_service.py | 2 +- .../test_local_lambda_invoke.py | 2 +- .../functional/local/lambdafn/test_runtime.py | 12 +++--- .../integration/buildcmd/build_integ_base.py | 6 +-- tests/integration/buildcmd/test_build_cmd.py | 6 +-- tests/integration/deprecation/__init__.py | 0 .../deprecation/test_deprecation_warning.py | 32 -------------- tests/integration/init/test_init_command.py | 3 +- .../local/invoke/invoke_integ_base.py | 6 +-- tests/integration/local/invoke/layer_utils.py | 6 +-- .../invoke/runtimes/test_with_runtime_zips.py | 8 +--- .../local/invoke/test_integrations_cli.py | 5 +-- .../local/start_api/start_api_integ_base.py | 6 +-- .../start_lambda_api_integ_base.py | 5 +-- .../publish/publish_app_integ_base.py | 6 +-- tests/integration/telemetry/integ_base.py | 8 +--- .../telemetry/test_installed_metric.py | 2 +- tests/unit/cli/test_command.py | 2 +- tests/unit/cli/test_context.py | 2 +- tests/unit/cli/test_global_config.py | 8 +--- tests/unit/cli/test_main.py | 26 +++++------ tests/unit/cli/test_types.py | 2 +- tests/unit/commands/_utils/test_options.py | 2 +- tests/unit/commands/_utils/test_template.py | 2 +- .../commands/buildcmd/test_build_context.py | 2 +- tests/unit/commands/buildcmd/test_command.py | 2 +- tests/unit/commands/init/test_cli.py | 2 +- .../local/cli_common/test_invoke_context.py | 2 +- .../generate_event/test_event_generation.py | 3 +- tests/unit/commands/local/invoke/test_cli.py | 2 +- .../commands/local/lib/swagger/test_parser.py | 2 +- .../commands/local/lib/swagger/test_reader.py | 2 +- .../commands/local/lib/test_api_provider.py | 2 +- .../local/lib/test_cfn_api_provider.py | 32 ++++++-------- .../local/lib/test_local_api_service.py | 2 +- .../commands/local/lib/test_local_lambda.py | 2 +- .../local/lib/test_local_lambda_service.py | 2 +- .../local/lib/test_sam_api_provider.py | 43 +++++++++---------- .../local/lib/test_sam_base_provider.py | 2 +- .../local/lib/test_sam_function_provider.py | 2 +- .../unit/commands/local/start_api/test_cli.py | 2 +- .../commands/local/start_lambda/test_cli.py | 2 +- tests/unit/commands/logs/test_command.py | 2 +- tests/unit/commands/logs/test_logs_context.py | 2 +- tests/unit/commands/publish/test_command.py | 2 +- tests/unit/commands/test_deploy.py | 2 +- tests/unit/commands/test_package.py | 2 +- .../lib/test_sam_template_validator.py | 2 +- tests/unit/commands/validate/test_cli.py | 2 +- .../unit/lib/build_module/test_app_builder.py | 9 +--- .../lib/build_module/test_workflow_config.py | 2 +- .../test_intrinsic_resolver.py | 9 +--- .../test_intrinsics_symbol_table.py | 2 +- tests/unit/lib/logs/test_fetcher.py | 2 +- tests/unit/lib/logs/test_formatter.py | 2 +- .../lib/samlib/test_cloudformation_command.py | 2 +- tests/unit/lib/telemetry/test_metrics.py | 2 +- tests/unit/lib/telemetry/test_telemetry.py | 2 +- tests/unit/lib/utils/test_codeuri.py | 5 +-- tests/unit/lib/utils/test_progressbar.py | 2 +- tests/unit/lib/utils/test_sam_logging.py | 2 +- tests/unit/lib/utils/test_stream_writer.py | 2 +- tests/unit/lib/utils/test_tar.py | 2 +- .../local/apigw/test_local_apigw_service.py | 2 +- .../apigw/test_service_error_responses.py | 2 +- tests/unit/local/docker/test_container.py | 2 +- .../docker/test_lambda_build_container.py | 8 +--- .../local/docker/test_lambda_container.py | 2 +- tests/unit/local/docker/test_lambda_image.py | 2 +- tests/unit/local/docker/test_manager.py | 2 +- tests/unit/local/docker/test_utils.py | 3 +- tests/unit/local/events/test_api_event.py | 2 +- tests/unit/local/init/test_init.py | 2 +- .../test_lambda_error_responses.py | 2 +- .../test_local_lambda_invoke_service.py | 2 +- tests/unit/local/lambdafn/test_config.py | 2 +- tests/unit/local/lambdafn/test_env_vars.py | 2 +- tests/unit/local/lambdafn/test_runtime.py | 2 +- tests/unit/local/lambdafn/test_zip.py | 2 +- .../unit/local/layers/test_download_layers.py | 8 +--- .../local/services/test_base_local_service.py | 2 +- 163 files changed, 312 insertions(+), 525 deletions(-) delete mode 100644 tests/integration/deprecation/__init__.py delete mode 100644 tests/integration/deprecation/test_deprecation_warning.py diff --git a/appveyor-windows.yml b/appveyor-windows.yml index 3243f006a6..0bddf396cf 100644 --- a/appveyor-windows.yml +++ b/appveyor-windows.yml @@ -25,6 +25,7 @@ environment: HOME: 'C:\Users\appveyor' HOMEDRIVE: 'C:' HOMEPATH: 'C:\Users\appveyor' + NOSE_PARAMETERIZED_NO_WARN: 1 init: # Uncomment this for RDP diff --git a/appveyor.yml b/appveyor.yml index 29b928b054..c7bf8d775c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -12,11 +12,13 @@ environment: - PYTHON_HOME: "C:\\Python36-x64" PYTHON_VERSION: '3.6.8' PYTHON_ARCH: '64' + NOSE_PARAMETERIZED_NO_WARN: 1 - PYTHON_HOME: "C:\\Python37-x64" PYTHON_VERSION: '3.7.4' PYTHON_ARCH: '64' RUN_SMOKE: 1 + NOSE_PARAMETERIZED_NO_WARN: 1 for: - @@ -30,12 +32,16 @@ for: - "echo %PATH%" - "python --version" # Upgrade setuptools, wheel and virtualenv + - "SET PATH=%PYTHON_HOME%;%PATH%" + - "echo %PYTHON_HOME%" + - "echo %PATH%" - "python -m pip install --upgrade setuptools wheel virtualenv" # Create new virtual environment and activate it - "rm -rf venv" - "python -m virtualenv venv" - "venv\\Scripts\\activate" + - "python --version" build_script: # Activate virtualenv again on windows diff --git a/pyproject.toml b/pyproject.toml index 40711e3163..a1b0857777 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ["setuptools", "wheel"] # PEP 508 specifications. [tool.black] line-length = 120 -target_version = ['py37', 'py27', 'py36'] +target_version = ['py37', 'py36'] exclude = ''' ( @@ -17,6 +17,7 @@ exclude = ''' | dist | pip-wheel-metadata | samcli/local/init/templates + | tests/integration/testdata )/ ) ''' diff --git a/pytest.ini b/pytest.ini index 926a10d096..d42cc7e27f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] env = AWS_DEFAULT_REGION = ap-southeast-1 -filterwarnings = - error +#filterwarnings = +# error diff --git a/requirements/base.txt b/requirements/base.txt index 39a0699fdf..0c4dac42ca 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,7 +1,5 @@ -six~=1.11.0 chevron~=0.12 click~=7.0 -enum34~=1.1.6; python_version<"3.4" Flask~=1.0.2 boto3~=1.9, >=1.9.56 PyYAML~=5.1 @@ -10,7 +8,6 @@ aws-sam-translator==1.15.1 docker~=4.0 dateparser~=0.7 python-dateutil~=2.6 -pathlib2~=2.3.2; python_version<"3.4" requests==2.22.0 serverlessrepo==0.1.9 aws_lambda_builders==0.5.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 029324c901..af29d59fa2 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,20 +1,11 @@ -coverage==4.3.4 -pytest-cov==2.4.0 -# astroid > 2.0.4 is not compatible with pylint1.7 -astroid>=1.5.8,<2.1.0 -pylint==1.7.2 +coverage==4.5.4 +pytest-cov==2.7.1 +pylint==2.3.1 # Test requirements -pytest==3.6.0 -py==1.5.1 -pluggy==0.6.0 -mock==2.0.0 -parameterized==0.6.1 -pathlib2==2.3.2; python_version<"3.4" -futures==3.2.0; python_version<"3.2.3" -# Py3.2 backport -backports.tempfile==1.0 -pytest-xdist==1.20.0 -pytest-forked==1.0.2 +pytest==5.2.1 +parameterized==0.7.0 +pytest-xdist==1.30.0 +pytest-forked==1.1.3 pytest-timeout==1.3.3 -pytest-rerunfailures==5.0 +pytest-rerunfailures==7.0 \ No newline at end of file diff --git a/samcli/cli/command.py b/samcli/cli/command.py index e7fca789ce..f56e93fb80 100644 --- a/samcli/cli/command.py +++ b/samcli/cli/command.py @@ -4,7 +4,6 @@ import logging import importlib -import sys from collections import OrderedDict import click @@ -24,13 +23,6 @@ "samcli.commands.publish", ] -DEPRECATION_NOTICE = ( - "Deprecated : AWS SAM CLI no longer supports " - "installations on Python 2.7. " - "Install AWS SAM CLI via https://docs.aws.amazon.com/serverless-application-model/" - "latest/developerguide/serverless-sam-cli-install.html for continued support with new versions. \n" -) - class BaseCommand(click.MultiCommand): """ @@ -53,7 +45,7 @@ class BaseCommand(click.MultiCommand): will produce a command name "baz". """ - def __init__(self, cmd_packages=None, *args, **kwargs): + def __init__(self, *args, cmd_packages=None, **kwargs): """ Initializes the class, optionally with a list of available commands @@ -69,9 +61,6 @@ def __init__(self, cmd_packages=None, *args, **kwargs): self._commands = {} self._commands = BaseCommand._set_commands(cmd_packages) - if sys.version_info.major == 2: - click.secho(DEPRECATION_NOTICE, fg="red", err=True) - @staticmethod def _set_commands(package_names): """ @@ -109,7 +98,7 @@ def get_command(self, ctx, cmd_name): """ if cmd_name not in self._commands: logger.error("Command %s not available", cmd_name) - return + return None pkg_name = self._commands[cmd_name] @@ -117,10 +106,10 @@ def get_command(self, ctx, cmd_name): mod = importlib.import_module(pkg_name) except ImportError: logger.exception("Command '%s' is not configured correctly. Unable to import '%s'", cmd_name, pkg_name) - return + return None if not hasattr(mod, "cli"): logger.error("Command %s is not configured correctly. It must expose an function called 'cli'", cmd_name) - return + return None return mod.cli diff --git a/samcli/cli/context.py b/samcli/cli/context.py index c50fefa8fe..f1d220598e 100644 --- a/samcli/cli/context.py +++ b/samcli/cli/context.py @@ -8,7 +8,7 @@ import click -class Context(object): +class Context: """ Top level context object for the CLI. Exposes common functionality required by a CLI, including logging, environment config parsing, debug logging etc. @@ -98,6 +98,8 @@ def command_path(self): if click_core_ctx: return click_core_ctx.command_path + return None + @staticmethod def get_current_context(): """ @@ -129,6 +131,8 @@ def my_command_handler(ctx): if click_core_ctx: return click_core_ctx.find_object(Context) or click_core_ctx.ensure_object(Context) + return None + def _refresh_session(self): """ Update boto3's default session by creating a new session based on values set in the context. Some properties of diff --git a/samcli/cli/global_config.py b/samcli/cli/global_config.py index 539aa2c1a0..dc0f208c9e 100644 --- a/samcli/cli/global_config.py +++ b/samcli/cli/global_config.py @@ -6,13 +6,10 @@ import logging import uuid import os +from pathlib import Path import click -try: - from pathlib import Path -except ImportError: # pragma: no cover - from pathlib2 import Path # pragma: no cover LOG = logging.getLogger(__name__) @@ -21,7 +18,7 @@ TELEMETRY_ENABLED_KEY = "telemetryEnabled" -class GlobalConfig(object): +class GlobalConfig: """ Contains helper methods for global configuration files and values. Handles configuration file creation, updates, and fetching in a platform-neutral way. diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py index 2b45e2e41e..370fac617d 100644 --- a/samcli/commands/_utils/template.py +++ b/samcli/commands/_utils/template.py @@ -3,14 +3,9 @@ """ import os -import six +import pathlib import yaml -try: - import pathlib -except ImportError: - import pathlib2 as pathlib - from samcli.yamlhelper import yaml_parse, yaml_dump @@ -218,7 +213,7 @@ def _resolve_relative_to(path, original_root, new_root): Updated path if the given path is a relative path. None, if the path is not a relative path. """ - if not isinstance(path, six.string_types) or path.startswith("s3://") or os.path.isabs(path): + if not isinstance(path, str) or path.startswith("s3://") or os.path.isabs(path): # Value is definitely NOT a relative path. It is either a S3 URi or Absolute path or not a string at all return None diff --git a/samcli/commands/build/build_context.py b/samcli/commands/build/build_context.py index 5892b8a6f8..8d4a166fdb 100644 --- a/samcli/commands/build/build_context.py +++ b/samcli/commands/build/build_context.py @@ -5,11 +5,7 @@ import logging import os import shutil - -try: - import pathlib -except ImportError: - import pathlib2 as pathlib +import pathlib from samcli.local.docker.manager import ContainerManager from samcli.commands.local.lib.sam_function_provider import SamFunctionProvider @@ -21,7 +17,7 @@ LOG = logging.getLogger(__name__) -class BuildContext(object): +class BuildContext: # Build directories need not be world writable. # This is usually a optimal permission for directories @@ -150,7 +146,7 @@ def functions_to_build(self): available_function_message = "{} not found. Possible options in your template: {}" \ .format(self._function_identifier, all_functions) LOG.info(available_function_message) - raise FunctionNotFound("Unable to find a Function with name '%s'", self._function_identifier) + raise FunctionNotFound("Unable to find a Function with name '{}'".format(self._function_identifier)) return [function] diff --git a/samcli/commands/build/exceptions.py b/samcli/commands/build/exceptions.py index 783a928803..8da0d4aa55 100644 --- a/samcli/commands/build/exceptions.py +++ b/samcli/commands/build/exceptions.py @@ -7,4 +7,3 @@ class InvalidBuildDirException(UserException): """ Value provided to --build-dir is invalid """ - pass diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 613b624ee1..d6f0d29595 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -14,7 +14,7 @@ @click.command( - "init", short_help="Init an AWS SAM application.", context_settings=dict(help_option_names=[u"-h", u"--help"]) + "init", short_help="Init an AWS SAM application.", context_settings=dict(help_option_names=["-h", "--help"]) ) @click.option("-l", "--location", help="Template location (git, mercurial, http(s), zip, path)") @click.option( diff --git a/samcli/commands/local/cli_common/invoke_context.py b/samcli/commands/local/cli_common/invoke_context.py index 8d870dbea0..6fdbb0584e 100644 --- a/samcli/commands/local/cli_common/invoke_context.py +++ b/samcli/commands/local/cli_common/invoke_context.py @@ -5,6 +5,7 @@ import errno import json import os +from pathlib import Path import samcli.lib.utils.osutils as osutils from samcli.lib.utils.stream_writer import StreamWriter @@ -18,16 +19,8 @@ from .user_exceptions import InvokeContextException, DebugContextException from ..lib.sam_function_provider import SamFunctionProvider -# This is an attempt to do a controlled import. pathlib is in the -# Python standard library starting at 3.4. This will import pathlib2, -# which is a backport of the Python Standard Library pathlib -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - -class InvokeContext(object): +class InvokeContext: """ Sets up a context to invoke Lambda functions locally by parsing all command line arguments necessary for the invoke. @@ -350,11 +343,10 @@ def _get_debug_context(debug_port, debug_args, debugger_path): except OSError as error: if error.errno == errno.ENOENT: raise DebugContextException("'{}' could not be found.".format(debugger_path)) - else: - raise error - # We turn off pylint here due to https://github.com/PyCQA/pylint/issues/1660 - if not debugger.is_dir(): # pylint: disable=no-member + raise error + + if not debugger.is_dir(): raise DebugContextException("'{}' should be a directory with the debugger in it.".format(debugger_path)) debugger_path = str(debugger) diff --git a/samcli/commands/local/cli_common/options.py b/samcli/commands/local/cli_common/options.py index 4d92c6dffc..d98633a418 100644 --- a/samcli/commands/local/cli_common/options.py +++ b/samcli/commands/local/cli_common/options.py @@ -1,14 +1,11 @@ """ Common CLI options for invoke command """ +from pathlib import Path import click -from samcli.commands._utils.options import template_click_option, docker_click_options, parameter_override_click_option -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from samcli.commands._utils.options import template_click_option, docker_click_options, parameter_override_click_option def get_application_dir(): diff --git a/samcli/commands/local/cli_common/user_exceptions.py b/samcli/commands/local/cli_common/user_exceptions.py index 7f474859a7..78d06fee00 100644 --- a/samcli/commands/local/cli_common/user_exceptions.py +++ b/samcli/commands/local/cli_common/user_exceptions.py @@ -10,68 +10,50 @@ class InvokeContextException(UserException): Something went wrong invoking the function. """ - pass - class InvalidSamTemplateException(UserException): """ The template provided was invalid and not able to transform into a Standard CloudFormation Template """ - pass - class SamTemplateNotFoundException(UserException): """ The SAM Template provided could not be found """ - pass - class DebugContextException(UserException): """ Something went wrong when creating the DebugContext """ - pass - class ImageBuildException(UserException): """ Image failed to build """ - pass - class CredentialsRequired(UserException): """ Credentials were not given when Required """ - pass - class ResourceNotFound(UserException): """ The Resource requested was not found """ - pass - class InvalidLayerVersionArn(UserException): """ The LayerVersion Arn given in the template is Invalid """ - pass - class UnsupportedIntrinsic(UserException): """ Value from a template has an Intrinsic that is unsupported """ - - pass diff --git a/samcli/commands/local/generate_event/cli.py b/samcli/commands/local/generate_event/cli.py index 89f911a661..2533d34715 100644 --- a/samcli/commands/local/generate_event/cli.py +++ b/samcli/commands/local/generate_event/cli.py @@ -32,4 +32,3 @@ def cli(self): """ Generate an event for one of the services listed below: """ - pass # pragma: no cover diff --git a/samcli/commands/local/lib/api_collector.py b/samcli/commands/local/lib/api_collector.py index 98fd2861c7..775fc36d10 100644 --- a/samcli/commands/local/lib/api_collector.py +++ b/samcli/commands/local/lib/api_collector.py @@ -14,7 +14,7 @@ LOG = logging.getLogger(__name__) -class ApiCollector(object): +class ApiCollector: def __init__(self): # Route properties stored per resource. self._route_per_resource = defaultdict(list) diff --git a/samcli/commands/local/lib/api_provider.py b/samcli/commands/local/lib/api_provider.py index 6e074bda02..dd244f6d7c 100644 --- a/samcli/commands/local/lib/api_provider.py +++ b/samcli/commands/local/lib/api_provider.py @@ -87,7 +87,8 @@ def find_api_provider(resources): for _, resource in resources.items(): if resource.get(CfnBaseApiProvider.RESOURCE_TYPE) in SamApiProvider.TYPES: return SamApiProvider() - elif resource.get(CfnBaseApiProvider.RESOURCE_TYPE) in CfnApiProvider.TYPES: + + if resource.get(CfnBaseApiProvider.RESOURCE_TYPE) in CfnApiProvider.TYPES: return CfnApiProvider() return SamApiProvider() diff --git a/samcli/commands/local/lib/cfn_api_provider.py b/samcli/commands/local/lib/cfn_api_provider.py index e844464eee..70ae3cbd43 100644 --- a/samcli/commands/local/lib/cfn_api_provider.py +++ b/samcli/commands/local/lib/cfn_api_provider.py @@ -211,3 +211,5 @@ def _get_integration_function_name(integration): if integration and isinstance(integration, dict): # Integration must be "aws_proxy" otherwise we don't care about it return LambdaUri.get_function_name(integration.get("Uri")) + + return None diff --git a/samcli/commands/local/lib/cfn_base_api_provider.py b/samcli/commands/local/lib/cfn_base_api_provider.py index d0a769e703..47bdf1ddf4 100644 --- a/samcli/commands/local/lib/cfn_base_api_provider.py +++ b/samcli/commands/local/lib/cfn_base_api_provider.py @@ -7,7 +7,7 @@ LOG = logging.getLogger(__name__) -class CfnBaseApiProvider(object): +class CfnBaseApiProvider: RESOURCE_TYPE = "Type" def extract_resources(self, resources, collector, cwd=None): diff --git a/samcli/commands/local/lib/debug_context.py b/samcli/commands/local/lib/debug_context.py index 26bb692452..053b580255 100644 --- a/samcli/commands/local/lib/debug_context.py +++ b/samcli/commands/local/lib/debug_context.py @@ -3,7 +3,7 @@ """ -class DebugContext(object): +class DebugContext: def __init__(self, debug_port=None, debugger_path=None, debug_args=None): self.debug_port = debug_port diff --git a/samcli/commands/local/lib/exceptions.py b/samcli/commands/local/lib/exceptions.py index ea5e9e9a31..e33a85bcad 100644 --- a/samcli/commands/local/lib/exceptions.py +++ b/samcli/commands/local/lib/exceptions.py @@ -8,16 +8,12 @@ class NoApisDefined(Exception): Raised when there are no APIs defined in the template """ - pass - class OverridesNotWellDefinedError(Exception): """ Raised when the overrides file is invalid """ - pass - class InvalidLayerReference(Exception): """ diff --git a/samcli/commands/local/lib/generated_sample_events/events.py b/samcli/commands/local/lib/generated_sample_events/events.py index a5d929401d..9afb858bcb 100644 --- a/samcli/commands/local/lib/generated_sample_events/events.py +++ b/samcli/commands/local/lib/generated_sample_events/events.py @@ -10,7 +10,7 @@ from chevron import renderer -class Events(object): +class Events: """ Events library class that loads and customizes event json files diff --git a/samcli/commands/local/lib/local_api_service.py b/samcli/commands/local/lib/local_api_service.py index d1ed8f7bca..8af4f784e5 100644 --- a/samcli/commands/local/lib/local_api_service.py +++ b/samcli/commands/local/lib/local_api_service.py @@ -12,7 +12,7 @@ LOG = logging.getLogger(__name__) -class LocalApiService(object): +class LocalApiService: """ Implementation of Local API service that is capable of serving API defined in a configuration file that invoke a Lambda function. @@ -129,3 +129,5 @@ def _make_static_dir_path(cwd, static_dir): if os.path.exists(static_dir_path): LOG.info("Mounting static files from %s at /", static_dir_path) return static_dir_path + + return None diff --git a/samcli/commands/local/lib/local_lambda.py b/samcli/commands/local/lib/local_lambda.py index 58a1eebb98..199f253a95 100644 --- a/samcli/commands/local/lib/local_lambda.py +++ b/samcli/commands/local/lib/local_lambda.py @@ -15,7 +15,7 @@ LOG = logging.getLogger(__name__) -class LocalLambdaRunner(object): +class LocalLambdaRunner: """ Runs Lambda functions locally. This class is a wrapper around the `samcli.local` library which takes care of actually running the function on a Docker container. @@ -86,7 +86,7 @@ def invoke(self, function_name, event, stdout=None, stderr=None): function_name, all_functions ) LOG.info(available_function_message) - raise FunctionNotFound("Unable to find a Function with name '%s'", function_name) + raise FunctionNotFound("Unable to find a Function with name '{}'".format(function_name)) LOG.debug("Found one Lambda function with name '%s'", function_name) diff --git a/samcli/commands/local/lib/local_lambda_service.py b/samcli/commands/local/lib/local_lambda_service.py index 45d3eb407b..07d11f78f0 100644 --- a/samcli/commands/local/lib/local_lambda_service.py +++ b/samcli/commands/local/lib/local_lambda_service.py @@ -8,7 +8,7 @@ LOG = logging.getLogger(__name__) -class LocalLambdaService(object): +class LocalLambdaService: """ Implementation of Local Lambda Invoke Service that is capable of serving the invoke path to your Lambda Functions that are defined in a SAM file. diff --git a/samcli/commands/local/lib/provider.py b/samcli/commands/local/lib/provider.py index 8aefb71107..b2b2eab1b3 100644 --- a/samcli/commands/local/lib/provider.py +++ b/samcli/commands/local/lib/provider.py @@ -5,8 +5,6 @@ import hashlib from collections import namedtuple -import six - from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn, UnsupportedIntrinsic # Named Tuple to representing the properties of a Lambda Function @@ -37,7 +35,7 @@ ) -class LayerVersion(object): +class LayerVersion: """ Represents the LayerVersion Resource for AWS Lambda """ @@ -54,7 +52,7 @@ def __init__(self, arn, codeuri): codeuri str CodeURI of the layer. This should contain the path to the layer code """ - if not isinstance(arn, six.string_types): + if not isinstance(arn, str): raise UnsupportedIntrinsic("{} is an Unsupported Intrinsic".format(arn)) self._arn = arn @@ -171,7 +169,7 @@ def __eq__(self, other): return False -class FunctionProvider(object): +class FunctionProvider: """ Abstract base class of the function provider. """ @@ -194,7 +192,7 @@ def get_all(self): raise NotImplementedError("not implemented") -class Api(object): +class Api: def __init__(self, routes=None): if routes is None: routes = [] @@ -259,7 +257,7 @@ def cors_to_headers(cors): return {h_key: h_value for h_key, h_value in headers.items() if h_value is not None} -class AbstractApiProvider(object): +class AbstractApiProvider: """ Abstract base class to return APIs and the functions they route to """ diff --git a/samcli/commands/local/lib/sam_base_provider.py b/samcli/commands/local/lib/sam_base_provider.py index 293724c640..268402d713 100644 --- a/samcli/commands/local/lib/sam_base_provider.py +++ b/samcli/commands/local/lib/sam_base_provider.py @@ -12,7 +12,7 @@ LOG = logging.getLogger(__name__) -class SamBaseProvider(object): +class SamBaseProvider: """ Base class for SAM Template providers """ diff --git a/samcli/commands/local/lib/sam_function_provider.py b/samcli/commands/local/lib/sam_function_provider.py index 4a0ef70f77..1e8365b2fa 100644 --- a/samcli/commands/local/lib/sam_function_provider.py +++ b/samcli/commands/local/lib/sam_function_provider.py @@ -3,7 +3,6 @@ """ import logging -import six from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn from .exceptions import InvalidLayerReference @@ -156,7 +155,7 @@ def _extract_sam_function_codeuri(name, resource_properties, code_property_key): """ codeuri = resource_properties.get(code_property_key, SamFunctionProvider._DEFAULT_CODEURI) # CodeUri can be a dictionary of S3 Bucket/Key or a S3 URI, neither of which are supported - if isinstance(codeuri, dict) or (isinstance(codeuri, six.string_types) and codeuri.startswith("s3://")): + if isinstance(codeuri, dict) or (isinstance(codeuri, str) and codeuri.startswith("s3://")): codeuri = SamFunctionProvider._DEFAULT_CODEURI LOG.warning( "Lambda function '%s' has specified S3 location for CodeUri which is unsupported. " @@ -253,7 +252,7 @@ def _parse_layer_info(list_of_layers, resources): ) # noqa: E501 # If the layer is a string, assume it is the arn - if isinstance(layer, six.string_types): + if isinstance(layer, str): layers.append(LayerVersion(layer, None)) continue diff --git a/samcli/commands/local/lib/swagger/integration_uri.py b/samcli/commands/local/lib/swagger/integration_uri.py index 437e8d6ed7..cab183bfb8 100644 --- a/samcli/commands/local/lib/swagger/integration_uri.py +++ b/samcli/commands/local/lib/swagger/integration_uri.py @@ -12,7 +12,7 @@ LOG = logging.getLogger(__name__) -class LambdaUri(object): +class LambdaUri: """ Purely static class that helps you parse Lambda Function Integration URI ARN """ @@ -161,12 +161,13 @@ def _get_function_name_from_arn(function_arn): LOG.debug("Stage variables are not supported. Ignoring integration with function ARN %s", function_arn) return None - elif re.match(LambdaUri._REGEX_VALID_FUNCTION_NAME, maybe_function_name): + if re.match(LambdaUri._REGEX_VALID_FUNCTION_NAME, maybe_function_name): # Yes, this is a real function name return maybe_function_name # Some unknown format LOG.debug("Ignoring integration ARN. Unable to parse Function Name from function arn %s", function_arn) + return None @staticmethod def _resolve_fn_sub(uri_data): diff --git a/samcli/commands/local/lib/swagger/parser.py b/samcli/commands/local/lib/swagger/parser.py index 9f5363570f..f5488aee4f 100644 --- a/samcli/commands/local/lib/swagger/parser.py +++ b/samcli/commands/local/lib/swagger/parser.py @@ -8,7 +8,7 @@ LOG = logging.getLogger(__name__) -class SwaggerParser(object): +class SwaggerParser: _INTEGRATION_KEY = "x-amazon-apigateway-integration" _ANY_METHOD_EXTENSION_KEY = "x-amazon-apigateway-any-method" _BINARY_MEDIA_TYPES_EXTENSION_KEY = "x-amazon-apigateway-binary-media-types" # pylint: disable=C0103 @@ -114,3 +114,5 @@ def _get_integration_function_name(self, method_config): if integration and isinstance(integration, dict) and integration.get("type") == IntegrationType.aws_proxy.value: # Integration must be "aws_proxy" otherwise we don't care about it return LambdaUri.get_function_name(integration.get("uri")) + + return None diff --git a/samcli/commands/local/lib/swagger/reader.py b/samcli/commands/local/lib/swagger/reader.py index 3ef417e858..264899db4e 100644 --- a/samcli/commands/local/lib/swagger/reader.py +++ b/samcli/commands/local/lib/swagger/reader.py @@ -43,10 +43,10 @@ def parse_aws_include_transform(data): """ if not data: - return + return None if _FN_TRANSFORM not in data: - return + return None transform_data = data[_FN_TRANSFORM] @@ -56,8 +56,10 @@ def parse_aws_include_transform(data): LOG.debug("Successfully parsed location from AWS::Include transform: %s", location) return location + return None -class SwaggerReader(object): + +class SwaggerReader: """ Class to read and parse Swagger document from a variety of sources. This class accepts the same data formats as available in Serverless::Api SAM resource @@ -152,7 +154,7 @@ def _download_swagger(self, location): """ if not location: - return + return None bucket, key, version = self._parse_s3_location(location) if bucket and key: @@ -163,7 +165,7 @@ def _download_swagger(self, location): if not isinstance(location, string_types): # This is not a string and not a S3 Location dictionary. Probably something invalid LOG.debug("Unable to download Swagger file. Invalid location: %s", location) - return + return None # ``location`` is a string and not a S3 path. It is probably a local path. Let's resolve relative path if any filepath = location @@ -173,7 +175,7 @@ def _download_swagger(self, location): if not os.path.exists(filepath): LOG.debug("Unable to download Swagger file. File not found at location %s", filepath) - return + return None LOG.debug("Reading Swagger document from local file at %s", filepath) with open(filepath, "r") as fp: diff --git a/samcli/commands/local/local.py b/samcli/commands/local/local.py index c24a54923f..663c1c4d87 100644 --- a/samcli/commands/local/local.py +++ b/samcli/commands/local/local.py @@ -16,7 +16,6 @@ def cli(): """ Run your Serverless application locally for quick development & testing """ - pass # pragma: no cover # Add individual commands under this group diff --git a/samcli/commands/logs/logs_context.py b/samcli/commands/logs/logs_context.py index b2cd8d5138..ef059eddf0 100644 --- a/samcli/commands/logs/logs_context.py +++ b/samcli/commands/logs/logs_context.py @@ -16,7 +16,7 @@ LOG = logging.getLogger(__name__) -class LogsCommandContext(object): +class LogsCommandContext: """ Sets up a context to run the Logs command by parsing the CLI arguments and creating necessary objects to be able to fetch and display logs @@ -209,7 +209,7 @@ def _parse_time(time_str, property_name): If the string cannot be parsed as a timestamp """ if not time_str: - return + return None parsed = parse_date(time_str) if not parsed: diff --git a/samcli/commands/publish/command.py b/samcli/commands/publish/command.py index d2817a2277..643895327e 100644 --- a/samcli/commands/publish/command.py +++ b/samcli/commands/publish/command.py @@ -5,12 +5,12 @@ import click import boto3 +from serverlessrepo.publish import CREATE_APPLICATION from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options from samcli.commands._utils.options import template_common_option from samcli.commands._utils.template import get_template_data from samcli.lib.telemetry.metrics import track_command -from serverlessrepo.publish import CREATE_APPLICATION LOG = logging.getLogger(__name__) diff --git a/samcli/commands/validate/lib/exceptions.py b/samcli/commands/validate/lib/exceptions.py index b6afd4b0f4..96aa3f6008 100644 --- a/samcli/commands/validate/lib/exceptions.py +++ b/samcli/commands/validate/lib/exceptions.py @@ -7,5 +7,3 @@ class InvalidSamDocumentException(Exception): """ Exception for Invalid Sam Documents """ - - pass diff --git a/samcli/commands/validate/lib/sam_template_validator.py b/samcli/commands/validate/lib/sam_template_validator.py index 6117273083..c869b53bf5 100644 --- a/samcli/commands/validate/lib/sam_template_validator.py +++ b/samcli/commands/validate/lib/sam_template_validator.py @@ -3,18 +3,18 @@ """ import logging import functools + from samtranslator.public.exceptions import InvalidDocumentException from samtranslator.parser import parser from samtranslator.translator.translator import Translator -from samcli.yamlhelper import yaml_dump -import six +from samcli.yamlhelper import yaml_dump from .exceptions import InvalidSamDocumentException LOG = logging.getLogger(__name__) -class SamTemplateValidator(object): +class SamTemplateValidator: def __init__(self, sam_template, managed_policy_loader): """ Construct a SamTemplateValidator @@ -113,7 +113,7 @@ def is_s3_uri(uri): Returns True if the uri given is an S3 uri, otherwise False """ - return isinstance(uri, six.string_types) and uri.startswith("s3://") + return isinstance(uri, str) and uri.startswith("s3://") @staticmethod def _update_to_s3_uri(property_key, resource_property_dict, s3_uri_value="s3://bucket/value"): diff --git a/samcli/commands/validate/validate.py b/samcli/commands/validate/validate.py index cdc9d7db71..4b554b0686 100644 --- a/samcli/commands/validate/validate.py +++ b/samcli/commands/validate/validate.py @@ -68,7 +68,7 @@ def _read_sam_file(template): click.secho("SAM Template Not Found", bg="red") raise SamTemplateNotFoundException("Template at {} is not found".format(template)) - with click.open_file(template, "r") as sam_template: + with click.open_file(template, "r", encoding="utf-8") as sam_template: sam_template = yaml_parse(sam_template.read()) return sam_template diff --git a/samcli/lib/build/app_builder.py b/samcli/lib/build/app_builder.py index 3ca2056769..186814f6bd 100644 --- a/samcli/lib/build/app_builder.py +++ b/samcli/lib/build/app_builder.py @@ -6,19 +6,15 @@ import io import json import logging - -try: - import pathlib -except ImportError: - import pathlib2 as pathlib +import pathlib import docker - -import samcli.lib.utils.osutils as osutils -from samcli.local.docker.lambda_build_container import LambdaBuildContainer from aws_lambda_builders.builder import LambdaBuilder from aws_lambda_builders.exceptions import LambdaBuilderError from aws_lambda_builders import RPC_PROTOCOL_VERSION as lambda_builders_protocol_version + +import samcli.lib.utils.osutils as osutils +from samcli.local.docker.lambda_build_container import LambdaBuildContainer from .workflow_config import get_workflow_config, supports_build_in_container @@ -41,7 +37,7 @@ class BuildError(Exception): pass -class ApplicationBuilder(object): +class ApplicationBuilder: """ Class to build an entire application. Currently, this class builds Lambda functions only, but there is nothing that is stopping this class from supporting other resource types. Building in context of Lambda functions refer to @@ -322,8 +318,7 @@ def _parse_builder_response(stdout_data, image_name): LOG.debug("Builder library does not support the supplied method") raise UnsupportedBuilderLibraryVersionError(image_name, msg) - else: - LOG.debug("Builder crashed") - raise ValueError(msg) + LOG.debug("Builder crashed") + raise ValueError(msg) return response diff --git a/samcli/lib/build/workflow_config.py b/samcli/lib/build/workflow_config.py index 475e036594..d649b80375 100644 --- a/samcli/lib/build/workflow_config.py +++ b/samcli/lib/build/workflow_config.py @@ -162,7 +162,7 @@ def _key(c): return True, None -class BasicWorkflowSelector(object): +class BasicWorkflowSelector: """ Basic workflow selector that returns the first available configuration in the given list of configurations """ diff --git a/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py b/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py index 0e99e90914..110d998448 100644 --- a/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py +++ b/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py @@ -27,7 +27,7 @@ LOG = logging.getLogger(__name__) -class IntrinsicResolver(object): +class IntrinsicResolver: AWS_INCLUDE = "AWS::Include" SUPPORTED_MACRO_TRANSFORMATIONS = [AWS_INCLUDE] _PSEUDO_REGEX = r"AWS::.*?" @@ -206,7 +206,8 @@ def intrinsic_property_resolver(self, intrinsic, ignore_errors, parent_function= if key in self.intrinsic_key_function_map: intrinsic_value = intrinsic.get(key) return self.intrinsic_key_function_map.get(key)(intrinsic_value, ignore_errors) - elif key in self.conditional_key_function_map: + + if key in self.conditional_key_function_map: intrinsic_value = intrinsic.get(key) return self.conditional_key_function_map.get(key)(intrinsic_value, ignore_errors) diff --git a/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py b/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py index 036deef72c..3a3725f0ef 100644 --- a/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py +++ b/samcli/lib/intrinsic_resolver/intrinsics_symbol_table.py @@ -12,7 +12,7 @@ LOG = logging.getLogger(__name__) -class IntrinsicsSymbolTable(object): +class IntrinsicsSymbolTable: AWS_ACCOUNT_ID = "AWS::AccountId" AWS_NOTIFICATION_ARN = "AWS::NotificationArn" AWS_PARTITION = "AWS::Partition" @@ -286,7 +286,7 @@ def get_translation(self, logical_id, resource_attributes=IntrinsicResolver.REF) """ logical_id_item = self.logical_id_translator.get(logical_id, {}) if any(isinstance(logical_id_item, object_type) for object_type in [string_types, list, bool, int]): - if resource_attributes != IntrinsicResolver.REF and resource_attributes != "": + if resource_attributes not in (IntrinsicResolver.REF, ""): return None return logical_id_item diff --git a/samcli/lib/logs/event.py b/samcli/lib/logs/event.py index ea95757252..73a2edf278 100644 --- a/samcli/lib/logs/event.py +++ b/samcli/lib/logs/event.py @@ -9,7 +9,7 @@ LOG = logging.getLogger(__name__) -class LogEvent(object): +class LogEvent: """ Data object representing a CloudWatch Log Event """ diff --git a/samcli/lib/logs/fetcher.py b/samcli/lib/logs/fetcher.py index 239fc01a65..1468eca98d 100644 --- a/samcli/lib/logs/fetcher.py +++ b/samcli/lib/logs/fetcher.py @@ -12,7 +12,7 @@ LOG = logging.getLogger(__name__) -class LogsFetcher(object): +class LogsFetcher: """ Fetch logs from a CloudWatch Logs group with the ability to scope to a particular time, filter by a pattern, and in the future possibly multiplex from from multiple streams together. diff --git a/samcli/lib/logs/formatter.py b/samcli/lib/logs/formatter.py index 09f15dc9a0..a5aab56b78 100644 --- a/samcli/lib/logs/formatter.py +++ b/samcli/lib/logs/formatter.py @@ -15,7 +15,7 @@ from builtins import map as imap -class LogsFormatter(object): +class LogsFormatter: """ Formats log messages returned by CloudWatch Logs service. """ @@ -121,7 +121,7 @@ def _pretty_print_event(event, colored): return " ".join([event.log_stream_name, event.timestamp, event.message]) -class LambdaLogMsgFormatters(object): +class LambdaLogMsgFormatters: """ Format logs printed by AWS Lambda functions. @@ -145,7 +145,7 @@ def colorize_errors(event, colored): return event -class KeywordHighlighter(object): +class KeywordHighlighter: """ Highlight certain keywords in the log line """ @@ -164,7 +164,7 @@ def highlight_keywords(self, event, colored): return event -class JSONMsgFormatter(object): +class JSONMsgFormatter: """ Pretty print JSONs within a message """ diff --git a/samcli/lib/logs/provider.py b/samcli/lib/logs/provider.py index 4a6d898e8e..90893e5238 100644 --- a/samcli/lib/logs/provider.py +++ b/samcli/lib/logs/provider.py @@ -3,7 +3,7 @@ """ -class LogGroupProvider(object): +class LogGroupProvider: """ Resolve the name of log group given the name of the resource """ diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index 541652cddf..0505103924 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -13,7 +13,7 @@ LOG = logging.getLogger(__name__) -class ResourceMetadataNormalizer(object): +class ResourceMetadataNormalizer: @staticmethod def normalize(template_dict): """ diff --git a/samcli/lib/samlib/wrapper.py b/samcli/lib/samlib/wrapper.py index 3a31e2c16e..141f1aeb0a 100644 --- a/samcli/lib/samlib/wrapper.py +++ b/samcli/lib/samlib/wrapper.py @@ -32,7 +32,7 @@ from .local_uri_plugin import SupportLocalUriPlugin -class SamTranslatorWrapper(object): +class SamTranslatorWrapper: _thisdir = os.path.dirname(os.path.abspath(__file__)) _DEFAULT_MANAGED_POLICIES_FILE = os.path.join(_thisdir, "default_managed_policies.json") @@ -116,7 +116,7 @@ def __managed_policy_map(self): raise ex -class _SamParserReimplemented(object): +class _SamParserReimplemented: """ Re-implementation (almost copy) of Parser class from SAM Translator """ diff --git a/samcli/lib/telemetry/telemetry.py b/samcli/lib/telemetry/telemetry.py index 9d25d08a5a..40a8f7cc7a 100644 --- a/samcli/lib/telemetry/telemetry.py +++ b/samcli/lib/telemetry/telemetry.py @@ -17,7 +17,7 @@ LOG = logging.getLogger(__name__) -class Telemetry(object): +class Telemetry: def __init__(self, url=None): """ Initialize the Telemetry object. @@ -109,6 +109,8 @@ def _default_session_id(self): if ctx: return ctx.session_id + return None + def _get_execution_environment(self): """ Returns the environment in which SAM CLI is running. Possible options are: diff --git a/samcli/lib/utils/colors.py b/samcli/lib/utils/colors.py index 19c7550494..84e3cbdbd7 100644 --- a/samcli/lib/utils/colors.py +++ b/samcli/lib/utils/colors.py @@ -5,7 +5,7 @@ import click -class Colored(object): +class Colored: """ Helper class to add ANSI colors and decorations to text. Given a string, ANSI colors are added with special prefix and suffix characters that are specially interpreted by Terminals to display colors. diff --git a/samcli/lib/utils/osutils.py b/samcli/lib/utils/osutils.py index d59e7ae43e..e22f3edbbb 100644 --- a/samcli/lib/utils/osutils.py +++ b/samcli/lib/utils/osutils.py @@ -49,16 +49,7 @@ def stdout(): io.BytesIO Byte stream of Stdout """ - - # We write all of the data to stdout with bytes, typically io.BytesIO. stdout in Python2 - # accepts bytes but Python3 does not. This is due to a type change on the attribute. To keep - # this consistent, we leave Python2 the same and get the .buffer attribute on stdout in Python3 - byte_stdout = sys.stdout - - if sys.version_info.major > 2: - byte_stdout = sys.stdout.buffer # pylint: disable=no-member - - return byte_stdout + return sys.stdout.buffer def stderr(): @@ -70,13 +61,4 @@ def stderr(): io.BytesIO Byte stream of stderr """ - - # We write all of the data to stderr with bytes, typically io.BytesIO. stderr in Python2 - # accepts bytes but Python3 does not. This is due to a type change on the attribute. To keep - # this consistent, we leave Python2 the same and get the .buffer attribute on stderr in Python3 - byte_stderr = sys.stderr - - if sys.version_info.major > 2: - byte_stderr = sys.stderr.buffer # pylint: disable=no-member - - return byte_stderr + return sys.stderr.buffer diff --git a/samcli/lib/utils/sam_logging.py b/samcli/lib/utils/sam_logging.py index 069933f10a..bf40d2ccaf 100644 --- a/samcli/lib/utils/sam_logging.py +++ b/samcli/lib/utils/sam_logging.py @@ -4,7 +4,7 @@ import logging -class SamCliLogger(object): +class SamCliLogger: @staticmethod def configure_logger(logger, formatter, level): """ diff --git a/samcli/lib/utils/stream_writer.py b/samcli/lib/utils/stream_writer.py index a931452e1e..da82625ce5 100644 --- a/samcli/lib/utils/stream_writer.py +++ b/samcli/lib/utils/stream_writer.py @@ -3,7 +3,7 @@ """ -class StreamWriter(object): +class StreamWriter: def __init__(self, stream, auto_flush=False): """ Instatiates new StreamWriter to the specified stream diff --git a/samcli/local/apigw/local_apigw_service.py b/samcli/local/apigw/local_apigw_service.py index db647ba6ff..e11191ec31 100644 --- a/samcli/local/apigw/local_apigw_service.py +++ b/samcli/local/apigw/local_apigw_service.py @@ -18,7 +18,7 @@ LOG = logging.getLogger(__name__) -class Route(object): +class Route: ANY_HTTP_METHODS = ["GET", "DELETE", "PUT", "POST", "HEAD", "OPTIONS", "PATCH"] def __init__(self, function_name, path, methods): @@ -260,7 +260,7 @@ def _parse_lambda_output(lambda_output, binary_types, flask_request): json_output = json.loads(lambda_output) if not isinstance(json_output, dict): - raise TypeError("Lambda returned %{s} instead of dict", type(json_output)) + raise TypeError("Lambda returned {} instead of dict".format(type(json_output))) status_code = json_output.get("statusCode") or 200 headers = LocalApigwService._merge_response_headers( diff --git a/samcli/local/apigw/path_converter.py b/samcli/local/apigw/path_converter.py index 274f4a9907..821657ef36 100644 --- a/samcli/local/apigw/path_converter.py +++ b/samcli/local/apigw/path_converter.py @@ -31,7 +31,7 @@ FLASK_TO_APIGW_REGEX = re.compile(FLASK_CAPTURE_ALL_PATH_REGEX) -class PathConverter(object): +class PathConverter: @staticmethod def convert_path_to_flask(path): """ diff --git a/samcli/local/apigw/service_error_responses.py b/samcli/local/apigw/service_error_responses.py index e32f6ec23a..e2214dada1 100644 --- a/samcli/local/apigw/service_error_responses.py +++ b/samcli/local/apigw/service_error_responses.py @@ -3,7 +3,7 @@ from flask import jsonify, make_response -class ServiceErrorResponses(object): +class ServiceErrorResponses: _NO_LAMBDA_INTEGRATION = {"message": "No function defined for resource method"} _MISSING_AUTHENTICATION = {"message": "Missing Authentication Token"} diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index b76c0ae9a9..3893b665e5 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -4,11 +4,7 @@ import itertools import os - -try: - import pathlib -except ImportError: - import pathlib2 as pathlib +import pathlib _init_path = str(pathlib.Path(os.path.dirname(__file__)).parent) _templates = os.path.join(_init_path, "init", "templates") @@ -78,13 +74,12 @@ ], } -SUPPORTED_DEP_MANAGERS = set( - [ - c["dependency_manager"] - for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))) - if c["dependency_manager"] - ] -) +SUPPORTED_DEP_MANAGERS = { + c["dependency_manager"] + for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))) + if c["dependency_manager"] +} + RUNTIMES = set( itertools.chain(*[c["runtimes"] for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))]) ) diff --git a/samcli/local/docker/container.py b/samcli/local/docker/container.py index d2910a936c..a2ffc15e77 100644 --- a/samcli/local/docker/container.py +++ b/samcli/local/docker/container.py @@ -14,7 +14,7 @@ LOG = logging.getLogger(__name__) -class Container(object): +class Container: """ Represents an instance of a Docker container with a specific configuration. The container is not actually created or executed until the appropriate methods are called. Each container instance is uniquely identified by an ID that diff --git a/samcli/local/docker/lambda_build_container.py b/samcli/local/docker/lambda_build_container.py index ed86f4fb87..3298941cd5 100644 --- a/samcli/local/docker/lambda_build_container.py +++ b/samcli/local/docker/lambda_build_container.py @@ -4,11 +4,7 @@ import json import logging - -try: - import pathlib -except ImportError: - import pathlib2 as pathlib +import pathlib from .container import Container diff --git a/samcli/local/docker/lambda_debug_entrypoint.py b/samcli/local/docker/lambda_debug_entrypoint.py index 17154e5394..4424942f6f 100644 --- a/samcli/local/docker/lambda_debug_entrypoint.py +++ b/samcli/local/docker/lambda_debug_entrypoint.py @@ -11,7 +11,7 @@ class DebuggingNotSupported(Exception): pass -class LambdaDebugEntryPoint(object): +class LambdaDebugEntryPoint: @staticmethod def get_entry_point(debug_port, debug_args_list, runtime, options): diff --git a/samcli/local/docker/lambda_image.py b/samcli/local/docker/lambda_image.py index 5a991f85fe..a8d9d418ed 100644 --- a/samcli/local/docker/lambda_image.py +++ b/samcli/local/docker/lambda_image.py @@ -1,21 +1,17 @@ """ Generates a Docker Image to be used for invoking a function locally """ -from enum import Enum import uuid import logging import hashlib +from enum import Enum +from pathlib import Path import docker from samcli.commands.local.cli_common.user_exceptions import ImageBuildException from samcli.lib.utils.tar import create_tarball -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - LOG = logging.getLogger(__name__) @@ -47,7 +43,7 @@ def has_value(cls, value): return any(value == item.value for item in cls) -class LambdaImage(object): +class LambdaImage: _LAYERS_DIR = "/opt" _DOCKER_LAMBDA_REPO_NAME = "lambci/lambda" _SAM_CLI_REPO_NAME = "samcli/lambda" diff --git a/samcli/local/docker/manager.py b/samcli/local/docker/manager.py index 0fb07c0391..39faf9296e 100644 --- a/samcli/local/docker/manager.py +++ b/samcli/local/docker/manager.py @@ -13,7 +13,7 @@ LOG = logging.getLogger(__name__) -class ContainerManager(object): +class ContainerManager: """ This class knows how to interface with Docker to create, execute and manage the container's life cycle. It can run multiple containers in parallel, and also comes with the ability to reuse existing containers in order to @@ -127,16 +127,16 @@ def pull_image(self, image_name, stream=None): raise DockerImagePullFailedException(str(ex)) # io streams, especially StringIO, work only with unicode strings - stream_writer.write(u"\nFetching {} Docker container image...".format(image_name)) + stream_writer.write("\nFetching {} Docker container image...".format(image_name)) # Each line contains information on progress of the pull. Each line is a JSON string for _ in result_itr: # For every line, print a dot to show progress - stream_writer.write(u".") + stream_writer.write(".") stream_writer.flush() # We are done. Go to the next line - stream_writer.write(u"\n") + stream_writer.write("\n") def has_image(self, image_name): """ diff --git a/samcli/local/docker/utils.py b/samcli/local/docker/utils.py index 2fadc38aa6..a67bc9e9fe 100644 --- a/samcli/local/docker/utils.py +++ b/samcli/local/docker/utils.py @@ -5,11 +5,7 @@ import os import re import posixpath - -try: - import pathlib -except ImportError: - import pathlib2 as pathlib +import pathlib def to_posix_path(code_path): diff --git a/samcli/local/events/api_event.py b/samcli/local/events/api_event.py index e17f125780..fa077e22f7 100644 --- a/samcli/local/events/api_event.py +++ b/samcli/local/events/api_event.py @@ -1,7 +1,7 @@ """Holds Classes for API Gateway to Lambda Events""" -class ContextIdentity(object): +class ContextIdentity: def __init__( self, api_key=None, @@ -62,7 +62,7 @@ def to_dict(self): return json_dict -class RequestContext(object): +class RequestContext: def __init__( self, resource_id="123456", @@ -128,7 +128,7 @@ def to_dict(self): return json_dict -class ApiGatewayLambdaEvent(object): +class ApiGatewayLambdaEvent: def __init__( self, http_method=None, diff --git a/samcli/local/lambda_service/lambda_error_responses.py b/samcli/local/lambda_service/lambda_error_responses.py index d003980042..49ee2c253f 100644 --- a/samcli/local/lambda_service/lambda_error_responses.py +++ b/samcli/local/lambda_service/lambda_error_responses.py @@ -6,7 +6,7 @@ from samcli.local.services.base_local_service import BaseLocalService -class LambdaErrorResponses(object): +class LambdaErrorResponses: # The content type of the Invoke request body is not JSON. UnsupportedMediaTypeException = ("UnsupportedMediaType", 415) diff --git a/samcli/local/lambda_service/local_lambda_invoke_service.py b/samcli/local/lambda_service/local_lambda_invoke_service.py index b41072e606..30c8aa9477 100644 --- a/samcli/local/lambda_service/local_lambda_invoke_service.py +++ b/samcli/local/lambda_service/local_lambda_invoke_service.py @@ -110,6 +110,8 @@ def validate_request(): "invocation-type: {} is not supported. RequestResponse is only supported.".format(invocation_type) ) + return None + def _construct_error_handling(self): """ Updates the Flask app with Error Handlers for different Error Codes diff --git a/samcli/local/lambdafn/config.py b/samcli/local/lambdafn/config.py index b7df4b5466..e438ab9355 100644 --- a/samcli/local/lambdafn/config.py +++ b/samcli/local/lambdafn/config.py @@ -5,7 +5,7 @@ from .env_vars import EnvironmentVariables -class FunctionConfig(object): +class FunctionConfig: """ Data class to store function configuration. This class is a flavor of function configuration passed to AWS Lambda APIs on the cloud. It is limited to properties that make sense in a local testing environment. diff --git a/samcli/local/lambdafn/env_vars.py b/samcli/local/lambdafn/env_vars.py index cf63bc35aa..3c2f6cabd2 100644 --- a/samcli/local/lambdafn/env_vars.py +++ b/samcli/local/lambdafn/env_vars.py @@ -5,7 +5,7 @@ import sys -class EnvironmentVariables(object): +class EnvironmentVariables: """ Use this class to get the environment variables necessary to run the Lambda function. It returns the AWS specific variables (credentials, regions, etc) along with any environment variables configured on the function. diff --git a/samcli/local/lambdafn/exceptions.py b/samcli/local/lambdafn/exceptions.py index 0fee680f43..438c331c3c 100644 --- a/samcli/local/lambdafn/exceptions.py +++ b/samcli/local/lambdafn/exceptions.py @@ -7,5 +7,3 @@ class FunctionNotFound(Exception): """ Raised when the requested Lambda function is not found """ - - pass diff --git a/samcli/local/lambdafn/runtime.py b/samcli/local/lambdafn/runtime.py index 0557fa781d..cc9116d522 100644 --- a/samcli/local/lambdafn/runtime.py +++ b/samcli/local/lambdafn/runtime.py @@ -16,7 +16,7 @@ LOG = logging.getLogger(__name__) -class LambdaRuntime(object): +class LambdaRuntime: """ This class represents a Local Lambda runtime. It can run the Lambda function code locally in a Docker container and return results. Public methods exposed by this class are similar to the AWS Lambda APIs, for convenience only. @@ -137,12 +137,13 @@ def signal_handler(sig, frame): if is_debugging: LOG.debug("Setting up SIGTERM interrupt handler") signal.signal(signal.SIGTERM, signal_handler) - else: - # Start a timer, we'll use this to abort the function if it runs beyond the specified timeout - LOG.debug("Starting a timer for %s seconds for function '%s'", timeout, function_name) - timer = threading.Timer(timeout, timer_handler, ()) - timer.start() - return timer + return None + + # Start a timer, we'll use this to abort the function if it runs beyond the specified timeout + LOG.debug("Starting a timer for %s seconds for function '%s'", timeout, function_name) + timer = threading.Timer(timeout, timer_handler, ()) + timer.start() + return timer @contextmanager def _get_code_dir(self, code_path): diff --git a/samcli/local/lambdafn/zip.py b/samcli/local/lambdafn/zip.py index 130e9569f7..b421451e6d 100644 --- a/samcli/local/lambdafn/zip.py +++ b/samcli/local/lambdafn/zip.py @@ -4,18 +4,14 @@ """ import os -import zipfile import logging +import zipfile +from pathlib import Path import requests from samcli.lib.utils.progressbar import progressbar -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - LOG = logging.getLogger(__name__) diff --git a/samcli/local/layers/layer_downloader.py b/samcli/local/layers/layer_downloader.py index 2744582c5b..6e6703e8e1 100644 --- a/samcli/local/layers/layer_downloader.py +++ b/samcli/local/layers/layer_downloader.py @@ -3,6 +3,7 @@ """ import logging +from pathlib import Path import boto3 from botocore.exceptions import NoCredentialsError, ClientError @@ -11,16 +12,11 @@ from samcli.local.lambdafn.zip import unzip_from_uri from samcli.commands.local.cli_common.user_exceptions import CredentialsRequired, ResourceNotFound -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - LOG = logging.getLogger(__name__) -class LayerDownloader(object): +class LayerDownloader: def __init__(self, layer_cache, cwd, lambda_client=None): """ @@ -98,8 +94,7 @@ def download(self, layer, force=False): layer.codeuri = resolve_code_path(self.cwd, layer.codeuri) return layer - # disabling no-member due to https://github.com/PyCQA/pylint/issues/1660 - layer_path = Path(self.layer_cache).resolve().joinpath(layer.name) # pylint: disable=no-member + layer_path = Path(self.layer_cache).resolve().joinpath(layer.name) is_layer_downloaded = self._is_layer_cached(layer_path) layer.codeuri = str(layer_path) diff --git a/samcli/local/services/base_local_service.py b/samcli/local/services/base_local_service.py index c0cea55a7d..0bdfd682b8 100644 --- a/samcli/local/services/base_local_service.py +++ b/samcli/local/services/base_local_service.py @@ -9,7 +9,7 @@ LOG = logging.getLogger(__name__) -class BaseLocalService(object): +class BaseLocalService: def __init__(self, is_debugging, port, host): """ Creates a BaseLocalService class @@ -78,7 +78,7 @@ def service_response(body, headers, status_code): return response -class LambdaOutputParser(object): +class LambdaOutputParser: @staticmethod def get_lambda_output(stdout_stream): """ diff --git a/samcli/yamlhelper.py b/samcli/yamlhelper.py index 658bb8183c..690e5a55b5 100644 --- a/samcli/yamlhelper.py +++ b/samcli/yamlhelper.py @@ -23,8 +23,6 @@ import yaml from yaml.resolver import ScalarNode, SequenceNode -import six - def intrinsics_multi_constructor(loader, tag_prefix, node): """ @@ -42,7 +40,7 @@ def intrinsics_multi_constructor(loader, tag_prefix, node): cfntag = prefix + tag - if tag == "GetAtt" and isinstance(node.value, six.string_types): + if tag == "GetAtt" and isinstance(node.value, str): # ShortHand notation for !GetAtt accepts Resource.Attribute format # while the standard notation is to use an array # [Resource, Attribute]. Convert shorthand to standard format diff --git a/scripts/check-isolated-needs-update.py b/scripts/check-isolated-needs-update.py index c4325f2e59..d4ad92ad2d 100644 --- a/scripts/check-isolated-needs-update.py +++ b/scripts/check-isolated-needs-update.py @@ -54,5 +54,5 @@ def get_requirements_list(content): assert installed_pkg_version in isolated_req_list, "{} is in base.txt but not in isolated.txt".format( installed_pkg_version ) - print ("{} is in the isolated.txt file".format(installed_pkg_version)) + print("{} is in the isolated.txt file".format(installed_pkg_version)) break diff --git a/scripts/check-requirements.py b/scripts/check-requirements.py index 07dd425e4d..4d4b522db7 100755 --- a/scripts/check-requirements.py +++ b/scripts/check-requirements.py @@ -23,9 +23,9 @@ def read(*filenames, **kwargs): if package.split("==")[0] not in exclude_packages: all_pkgs_list.append(package) all_pkgs_list = sorted(all_pkgs_list) -print ("installed package/versions" + os.linesep) -print (",".join(all_pkgs_list)) -print (os.linesep) +print("installed package/versions" + os.linesep) +print(",".join(all_pkgs_list)) +print(os.linesep) content = read(os.path.join("requirements", "isolated.txt")) @@ -35,9 +35,9 @@ def read(*filenames, **kwargs): locked_pkgs.append(line) locked_pkgs = sorted(locked_pkgs) -print ("locked package/versions" + os.linesep) -print (",".join(locked_pkgs)) -print (os.linesep) +print("locked package/versions" + os.linesep) +print(",".join(locked_pkgs)) +print(os.linesep) assert len(locked_pkgs) == len(all_pkgs_list), "Number of expected dependencies do not match the number installed" assert locked_pkgs == all_pkgs_list, "The list of expected dependencies do not match what is installed" diff --git a/setup.py b/setup.py index 77265e87df..9c118def11 100644 --- a/setup.py +++ b/setup.py @@ -43,8 +43,8 @@ def read_version(): license="Apache License 2.0", packages=find_packages(exclude=["tests.*", "tests"]), keywords="AWS SAM CLI", - # Support Python 2.7 and 3.6 or greater - python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*", + # Support Python 3.6 or greater + python_requires=">=3.6, <=4.0, !=4.0", entry_points={"console_scripts": ["{}=samcli.cli.main:cli".format(cmd_name)]}, install_requires=read_requirements("base.txt"), extras_require={"dev": read_requirements("dev.txt")}, @@ -58,7 +58,6 @@ def read_version(): "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Topic :: Internet", diff --git a/tests/functional/commands/cli/test_global_config.py b/tests/functional/commands/cli/test_global_config.py index c4c92bcf58..ed550f762a 100644 --- a/tests/functional/commands/cli/test_global_config.py +++ b/tests/functional/commands/cli/test_global_config.py @@ -2,15 +2,11 @@ import tempfile import shutil -from mock import mock_open, patch +from unittest.mock import mock_open, patch from unittest import TestCase from json import JSONDecodeError from samcli.cli.global_config import GlobalConfig - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class TestGlobalConfig(TestCase): diff --git a/tests/functional/commands/local/lib/test_local_api_service.py b/tests/functional/commands/local/lib/test_local_api_service.py index 50131b2f1c..35077aab07 100644 --- a/tests/functional/commands/local/lib/test_local_api_service.py +++ b/tests/functional/commands/local/lib/test_local_api_service.py @@ -22,7 +22,7 @@ from tests.functional.function_code import nodejs_lambda, API_GATEWAY_ECHO_EVENT from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch logging.basicConfig(level=logging.INFO) diff --git a/tests/functional/commands/local/lib/test_local_lambda.py b/tests/functional/commands/local/lib/test_local_lambda.py index 19d1c1569a..55fa875c18 100644 --- a/tests/functional/commands/local/lib/test_local_lambda.py +++ b/tests/functional/commands/local/lib/test_local_lambda.py @@ -18,7 +18,7 @@ from tests.functional.function_code import nodejs_lambda, GET_ENV_VAR from unittest import TestCase -from mock import Mock +from unittest.mock import Mock logging.basicConfig(level=logging.INFO) diff --git a/tests/functional/commands/validate/lib/test_sam_template_validator.py b/tests/functional/commands/validate/lib/test_sam_template_validator.py index f421a0d4ec..a9610e46e8 100644 --- a/tests/functional/commands/validate/lib/test_sam_template_validator.py +++ b/tests/functional/commands/validate/lib/test_sam_template_validator.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from parameterized import parameterized import samcli.yamlhelper as yamlhelper diff --git a/tests/functional/local/apigw/test_local_apigw_service.py b/tests/functional/local/apigw/test_local_apigw_service.py index 78b9c6b47a..38bd9b3cfe 100644 --- a/tests/functional/local/apigw/test_local_apigw_service.py +++ b/tests/functional/local/apigw/test_local_apigw_service.py @@ -7,7 +7,7 @@ import requests import random -from mock import Mock +from unittest.mock import Mock from samcli.local.apigw.local_apigw_service import Route, LocalApigwService from tests.functional.function_code import ( diff --git a/tests/functional/local/lambda_service/test_local_lambda_invoke.py b/tests/functional/local/lambda_service/test_local_lambda_invoke.py index 4c4489f1df..a4db580f55 100644 --- a/tests/functional/local/lambda_service/test_local_lambda_invoke.py +++ b/tests/functional/local/lambda_service/test_local_lambda_invoke.py @@ -1,7 +1,7 @@ import threading import shutil import random -from mock import Mock +from unittest.mock import Mock import time from unittest import TestCase import os diff --git a/tests/functional/local/lambdafn/test_runtime.py b/tests/functional/local/lambdafn/test_runtime.py index 57f4dd4a4f..4f0ce0152d 100644 --- a/tests/functional/local/lambdafn/test_runtime.py +++ b/tests/functional/local/lambdafn/test_runtime.py @@ -95,7 +95,7 @@ def test_function_timeout(self): # Make sure that the wall clock duration is around the ballpark of timeout value wall_clock_func_duration = end - start - print ("Function completed in {} seconds".format(wall_clock_func_duration)) + print("Function completed in {} seconds".format(wall_clock_func_duration)) # The function should *not* preemptively stop self.assertGreater(wall_clock_func_duration, timeout - 1) # The function should not run for much longer than timeout. @@ -214,7 +214,7 @@ def tearDown(self): def _invoke_sleep(self, timeout, sleep_duration, check_stdout, exceptions=None): name = "sleepfunction_timeout_{}_sleep_{}".format(timeout, sleep_duration) - print ("Invoking function " + name) + print("Invoking function " + name) try: stdout_stream = io.BytesIO() stdout_stream_writer = StreamWriter(stdout_stream) @@ -276,10 +276,10 @@ def test_parallel(self): t.join() for e in exceptions: - print ("-------------") - print ("ERROR in function " + e["name"]) - print (e["error"]) - print ("-------------") + print("-------------") + print("ERROR in function " + e["name"]) + print(e["error"]) + print("-------------") if len(exceptions) > 0: raise AssertionError("Test failed. See print outputs above for details on the thread that failed") diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 8b69f94726..8a393d8dca 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -9,11 +9,7 @@ from unittest import TestCase import docker - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path from samcli.yamlhelper import yaml_parse from tests.testing_utils import IS_WINDOWS diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index fef5a26dea..5bc05fe315 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -3,11 +3,7 @@ import subprocess import logging from unittest import skipIf - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path from parameterized import parameterized from .build_integ_base import BuildIntegBase diff --git a/tests/integration/deprecation/__init__.py b/tests/integration/deprecation/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/integration/deprecation/test_deprecation_warning.py b/tests/integration/deprecation/test_deprecation_warning.py deleted file mode 100644 index d4ea9e8c56..0000000000 --- a/tests/integration/deprecation/test_deprecation_warning.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import re -import subprocess -import sys - -from unittest import TestCase - -from samcli.cli.command import DEPRECATION_NOTICE - - -class TestPy2DeprecationWarning(TestCase): - def base_command(self): - command = "sam" - if os.getenv("SAM_CLI_DEV"): - command = "samdev" - - return command - - def run_cmd(self): - # Checking with base command to see if warning is present if running in python2 - cmd_list = [self.base_command()] - process = subprocess.Popen(cmd_list, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - return process - - def test_print_deprecation_warning_if_py2(self): - process = self.run_cmd() - (stdoutdata, stderrdata) = process.communicate() - - expected_notice = re.sub(r"\n", os.linesep, DEPRECATION_NOTICE) - # Deprecation notice should be part of the command output if running in python 2 - if sys.version_info.major == 2: - self.assertIn(expected_notice, stderrdata.decode()) diff --git a/tests/integration/init/test_init_command.py b/tests/integration/init/test_init_command.py index 79acb2527c..138c22e862 100644 --- a/tests/integration/init/test_init_command.py +++ b/tests/integration/init/test_init_command.py @@ -1,8 +1,7 @@ from unittest import TestCase from subprocess import Popen import os - -from backports import tempfile +import tempfile class TestBasicInitCommand(TestCase): diff --git a/tests/integration/local/invoke/invoke_integ_base.py b/tests/integration/local/invoke/invoke_integ_base.py index fe13436acb..c941967ae4 100644 --- a/tests/integration/local/invoke/invoke_integ_base.py +++ b/tests/integration/local/invoke/invoke_integ_base.py @@ -1,10 +1,6 @@ import os from unittest import TestCase - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class InvokeIntegBase(TestCase): diff --git a/tests/integration/local/invoke/layer_utils.py b/tests/integration/local/invoke/layer_utils.py index eb9f135002..0743cd7084 100644 --- a/tests/integration/local/invoke/layer_utils.py +++ b/tests/integration/local/invoke/layer_utils.py @@ -4,11 +4,7 @@ import boto3 from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class LayerUtils(object): diff --git a/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py b/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py index 0f1290c711..3b6177e1ad 100644 --- a/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py +++ b/tests/integration/local/invoke/runtimes/test_with_runtime_zips.py @@ -8,11 +8,7 @@ import pytest from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class TestWithDifferentLambdaRuntimeZips(InvokeIntegBase): @@ -58,4 +54,4 @@ def test_custom_provided_runtime(self): self.assertEqual(return_code, 0) process_stdout = b"".join(process.stdout.readlines()).strip() - self.assertEqual(process_stdout.decode("utf-8"), u'{"body":"hello 曰有冥 world 🐿","statusCode":200,"headers":{}}') + self.assertEqual(process_stdout.decode("utf-8"), '{"body":"hello 曰有冥 world 🐿","statusCode":200,"headers":{}}') diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index 2042e5cf95..203ebb7797 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -19,10 +19,7 @@ # This is to restrict layers tests to run outside of Travis and when the branch is not master. SKIP_LAYERS_TESTS = RUNNING_ON_CI and RUNNING_TEST_FOR_MASTER_ON_CI -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class TestSamPython36HelloWorldIntegration(InvokeIntegBase): diff --git a/tests/integration/local/start_api/start_api_integ_base.py b/tests/integration/local/start_api/start_api_integ_base.py index a212eb1036..e4b32610a7 100644 --- a/tests/integration/local/start_api/start_api_integ_base.py +++ b/tests/integration/local/start_api/start_api_integ_base.py @@ -4,11 +4,7 @@ import time import os import random - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class StartApiIntegBaseClass(TestCase): diff --git a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py index 7fd5099f0a..e0a72ec82a 100644 --- a/tests/integration/local/start_lambda/start_lambda_api_integ_base.py +++ b/tests/integration/local/start_lambda/start_lambda_api_integ_base.py @@ -5,10 +5,7 @@ import os import random -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class StartLambdaIntegBaseClass(TestCase): diff --git a/tests/integration/publish/publish_app_integ_base.py b/tests/integration/publish/publish_app_integ_base.py index a937149a41..1b430a2375 100644 --- a/tests/integration/publish/publish_app_integ_base.py +++ b/tests/integration/publish/publish_app_integ_base.py @@ -7,11 +7,7 @@ from unittest import TestCase import boto3 - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class PublishAppIntegBase(TestCase): diff --git a/tests/integration/telemetry/integ_base.py b/tests/integration/telemetry/integ_base.py index 314da3b181..6637e7bca6 100644 --- a/tests/integration/telemetry/integ_base.py +++ b/tests/integration/telemetry/integ_base.py @@ -12,11 +12,7 @@ from threading import Thread from collections import deque from unittest import TestCase - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path from samcli.cli.global_config import GlobalConfig from samcli.cli.main import TELEMETRY_PROMPT @@ -196,5 +192,5 @@ def _request_handler(self, **kwargs): def _shutdown_flask(self): # Based on http://flask.pocoo.org/snippets/67/ request.environ.get("werkzeug.server.shutdown")() - print ("Server shutting down...") + print("Server shutting down...") return "" diff --git a/tests/integration/telemetry/test_installed_metric.py b/tests/integration/telemetry/test_installed_metric.py index bc1518bf21..fa4d3b679b 100644 --- a/tests/integration/telemetry/test_installed_metric.py +++ b/tests/integration/telemetry/test_installed_metric.py @@ -1,6 +1,6 @@ import platform -from mock import ANY +from unittest.mock import ANY from .integ_base import IntegBase, TelemetryServer, EXPECTED_TELEMETRY_PROMPT from samcli import __version__ as SAM_CLI_VERSION diff --git a/tests/unit/cli/test_command.py b/tests/unit/cli/test_command.py index 082af8bc8c..4c51d35c0f 100644 --- a/tests/unit/cli/test_command.py +++ b/tests/unit/cli/test_command.py @@ -1,7 +1,7 @@ import click from unittest import TestCase -from mock import Mock, patch, call +from unittest.mock import Mock, patch, call from samcli.cli.command import BaseCommand diff --git a/tests/unit/cli/test_context.py b/tests/unit/cli/test_context.py index 7bd04e4ca9..403df2539b 100644 --- a/tests/unit/cli/test_context.py +++ b/tests/unit/cli/test_context.py @@ -2,7 +2,7 @@ import logging from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.cli.context import Context diff --git a/tests/unit/cli/test_global_config.py b/tests/unit/cli/test_global_config.py index 776952dbfd..5432488303 100644 --- a/tests/unit/cli/test_global_config.py +++ b/tests/unit/cli/test_global_config.py @@ -1,12 +1,8 @@ -from mock import mock_open, patch, Mock +from unittest.mock import mock_open, patch, Mock from unittest import TestCase from parameterized import parameterized from samcli.cli.global_config import GlobalConfig - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path class TestGlobalConfig(TestCase): diff --git a/tests/unit/cli/test_main.py b/tests/unit/cli/test_main.py index 486d4154f1..e24cc900ea 100644 --- a/tests/unit/cli/test_main.py +++ b/tests/unit/cli/test_main.py @@ -1,4 +1,4 @@ -import mock +from unittest.mock import patch, Mock, PropertyMock from unittest import TestCase from click.testing import CliRunner @@ -11,8 +11,8 @@ def test_cli_base(self): Just invoke the CLI without any commands and assert that help text was printed :return: """ - mock_cfg = mock.Mock() - with mock.patch("samcli.cli.main.global_cfg", mock_cfg): + mock_cfg = Mock() + with patch("samcli.cli.main.global_cfg", mock_cfg): runner = CliRunner() result = runner.invoke(cli, []) self.assertEqual(result.exit_code, 0) @@ -21,25 +21,23 @@ def test_cli_base(self): def test_cli_some_command(self): - mock_cfg = mock.Mock() - with mock.patch("samcli.cli.main.global_cfg", mock_cfg): + mock_cfg = Mock() + with patch("samcli.cli.main.global_cfg", mock_cfg): runner = CliRunner() result = runner.invoke(cli, ["local", "generate-event", "s3"]) self.assertEqual(result.exit_code, 0) def test_cli_with_debug(self): - mock_cfg = mock.Mock() - with mock.patch("samcli.cli.main.global_cfg", mock_cfg): + mock_cfg = Mock() + with patch("samcli.cli.main.global_cfg", mock_cfg): runner = CliRunner() result = runner.invoke(cli, ["local", "generate-event", "s3", "put", "--debug"]) self.assertEqual(result.exit_code, 0) - @mock.patch("samcli.cli.main.send_installed_metric") + @patch("samcli.cli.main.send_installed_metric") def test_cli_enable_telemetry_with_prompt(self, send_installed_metric_mock): - with mock.patch( - "samcli.cli.global_config.GlobalConfig.telemetry_enabled", new_callable=mock.PropertyMock - ) as mock_flag: + with patch("samcli.cli.global_config.GlobalConfig.telemetry_enabled", new_callable=PropertyMock) as mock_flag: mock_flag.return_value = None runner = CliRunner() runner.invoke(cli, ["local", "generate-event", "s3"]) @@ -48,11 +46,9 @@ def test_cli_enable_telemetry_with_prompt(self, send_installed_metric_mock): # If telemetry is enabled, this should be called send_installed_metric_mock.assert_called_once() - @mock.patch("samcli.cli.main.send_installed_metric") + @patch("samcli.cli.main.send_installed_metric") def test_prompt_skipped_when_value_set(self, send_installed_metric_mock): - with mock.patch( - "samcli.cli.global_config.GlobalConfig.telemetry_enabled", new_callable=mock.PropertyMock - ) as mock_flag: + with patch("samcli.cli.global_config.GlobalConfig.telemetry_enabled", new_callable=PropertyMock) as mock_flag: mock_flag.return_value = True runner = CliRunner() runner.invoke(cli, ["local", "generate-event", "s3"]) diff --git a/tests/unit/cli/test_types.py b/tests/unit/cli/test_types.py index a46b72f9d1..1251392c88 100644 --- a/tests/unit/cli/test_types.py +++ b/tests/unit/cli/test_types.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, ANY +from unittest.mock import Mock, ANY from nose_parameterized import parameterized from samcli.cli.types import CfnParameterOverridesType diff --git a/tests/unit/commands/_utils/test_options.py b/tests/unit/commands/_utils/test_options.py index 44bad654fb..43276c824c 100644 --- a/tests/unit/commands/_utils/test_options.py +++ b/tests/unit/commands/_utils/test_options.py @@ -5,7 +5,7 @@ import os from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.commands._utils.options import get_or_default_template_file_name, _TEMPLATE_OPTION_DEFAULT_VALUE diff --git a/tests/unit/commands/_utils/test_template.py b/tests/unit/commands/_utils/test_template.py index 6d1b8575e6..04cd9b0b5c 100644 --- a/tests/unit/commands/_utils/test_template.py +++ b/tests/unit/commands/_utils/test_template.py @@ -3,7 +3,7 @@ import yaml from unittest import TestCase -from mock import patch, mock_open +from unittest.mock import patch, mock_open from parameterized import parameterized, param from samcli.commands._utils.template import ( diff --git a/tests/unit/commands/buildcmd/test_build_context.py b/tests/unit/commands/buildcmd/test_build_context.py index 7d087336b7..10be5be039 100644 --- a/tests/unit/commands/buildcmd/test_build_context.py +++ b/tests/unit/commands/buildcmd/test_build_context.py @@ -1,6 +1,6 @@ import os from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from samcli.commands.build.build_context import BuildContext from samcli.commands.build.exceptions import InvalidBuildDirException diff --git a/tests/unit/commands/buildcmd/test_command.py b/tests/unit/commands/buildcmd/test_command.py index b609ad0402..d6880955c6 100644 --- a/tests/unit/commands/buildcmd/test_command.py +++ b/tests/unit/commands/buildcmd/test_command.py @@ -2,7 +2,7 @@ import click from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from parameterized import parameterized from samcli.commands.build.command import do_cli, _get_mode_value_from_envvar diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index 4034d5b10d..2754fe6352 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.commands.init import do_cli as init_cli from samcli.local.init.exceptions import GenerateProjectFailedError diff --git a/tests/unit/commands/local/cli_common/test_invoke_context.py b/tests/unit/commands/local/cli_common/test_invoke_context.py index 14e60e9030..ccc95d112b 100644 --- a/tests/unit/commands/local/cli_common/test_invoke_context.py +++ b/tests/unit/commands/local/cli_common/test_invoke_context.py @@ -8,7 +8,7 @@ from samcli.commands.local.cli_common.invoke_context import InvokeContext from unittest import TestCase -from mock import Mock, PropertyMock, patch, ANY, mock_open +from unittest.mock import Mock, PropertyMock, patch, ANY, mock_open class TestInvokeContext__enter__(TestCase): diff --git a/tests/unit/commands/local/generate_event/test_event_generation.py b/tests/unit/commands/local/generate_event/test_event_generation.py index 1b5a5ea730..d719fb45db 100644 --- a/tests/unit/commands/local/generate_event/test_event_generation.py +++ b/tests/unit/commands/local/generate_event/test_event_generation.py @@ -1,8 +1,7 @@ import os from unittest import TestCase -from mock import Mock -from mock import patch +from unittest.mock import Mock, patch from samcli.commands.local.lib.generated_sample_events import events from samcli.commands.local.generate_event.event_generation import ServiceCommand diff --git a/tests/unit/commands/local/invoke/test_cli.py b/tests/unit/commands/local/invoke/test_cli.py index a15448833b..d38e12b338 100644 --- a/tests/unit/commands/local/invoke/test_cli.py +++ b/tests/unit/commands/local/invoke/test_cli.py @@ -3,7 +3,7 @@ """ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from parameterized import parameterized, param from samcli.local.lambdafn.exceptions import FunctionNotFound diff --git a/tests/unit/commands/local/lib/swagger/test_parser.py b/tests/unit/commands/local/lib/swagger/test_parser.py index f5cf36bce9..6468332db8 100644 --- a/tests/unit/commands/local/lib/swagger/test_parser.py +++ b/tests/unit/commands/local/lib/swagger/test_parser.py @@ -3,7 +3,7 @@ """ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from parameterized import parameterized, param from samcli.commands.local.lib.swagger.parser import SwaggerParser diff --git a/tests/unit/commands/local/lib/swagger/test_reader.py b/tests/unit/commands/local/lib/swagger/test_reader.py index 9e71ce7bdc..81d52561bd 100644 --- a/tests/unit/commands/local/lib/swagger/test_reader.py +++ b/tests/unit/commands/local/lib/swagger/test_reader.py @@ -5,7 +5,7 @@ from unittest import TestCase from parameterized import parameterized, param -from mock import Mock, patch +from unittest.mock import Mock, patch from samcli.commands.local.lib.swagger.reader import parse_aws_include_transform, SwaggerReader diff --git a/tests/unit/commands/local/lib/test_api_provider.py b/tests/unit/commands/local/lib/test_api_provider.py index 426fa37d05..3fcc0404ed 100644 --- a/tests/unit/commands/local/lib/test_api_provider.py +++ b/tests/unit/commands/local/lib/test_api_provider.py @@ -1,7 +1,7 @@ from collections import OrderedDict from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.commands.local.lib.provider import Api from samcli.commands.local.lib.api_provider import ApiProvider diff --git a/tests/unit/commands/local/lib/test_cfn_api_provider.py b/tests/unit/commands/local/lib/test_cfn_api_provider.py index cbf5d40d2e..c0cda6959a 100644 --- a/tests/unit/commands/local/lib/test_cfn_api_provider.py +++ b/tests/unit/commands/local/lib/test_cfn_api_provider.py @@ -3,8 +3,7 @@ from collections import OrderedDict from unittest import TestCase -from mock import patch -from six import assertCountEqual +from unittest.mock import patch from samcli.commands.local.lib.api_provider import ApiProvider from samcli.commands.local.lib.cfn_api_provider import CfnApiProvider @@ -36,7 +35,7 @@ def test_with_inline_swagger_apis(self): } provider = ApiProvider(template) - assertCountEqual(self, self.input_routes, provider.routes) + self.assertCountEqual(self.input_routes, provider.routes) def test_with_swagger_as_local_file(self): with tempfile.NamedTemporaryFile(mode="w", delete=False) as fp: @@ -52,7 +51,7 @@ def test_with_swagger_as_local_file(self): } provider = ApiProvider(template) - assertCountEqual(self, self.input_routes, provider.routes) + self.assertCountEqual(self.input_routes, provider.routes) def test_body_with_swagger_as_local_file_expect_fail(self): with tempfile.NamedTemporaryFile(mode="w", delete=False) as fp: @@ -81,7 +80,7 @@ def test_with_swagger_as_both_body_and_uri_called(self, SwaggerReaderMock): cwd = "foo" provider = ApiProvider(template, cwd=cwd) - assertCountEqual(self, self.input_routes, provider.routes) + self.assertCountEqual(self.input_routes, provider.routes) SwaggerReaderMock.assert_called_with(definition_body=body, definition_uri=filename, working_dir=cwd) def test_swagger_with_any_method(self): @@ -100,7 +99,7 @@ def test_swagger_with_any_method(self): } provider = ApiProvider(template) - assertCountEqual(self, expected_routes, provider.routes) + self.assertCountEqual(expected_routes, provider.routes) def test_with_binary_media_types(self): template = { @@ -120,8 +119,8 @@ def test_with_binary_media_types(self): ] provider = ApiProvider(template) - assertCountEqual(self, expected_apis, provider.routes) - assertCountEqual(self, provider.api.binary_media_types, expected_binary_types) + self.assertCountEqual(expected_apis, provider.routes) + self.assertCountEqual(provider.api.binary_media_types, expected_binary_types) def test_with_binary_media_types_in_swagger_and_on_resource(self): input_routes = [Route(path="/path", methods=["OPTIONS"], function_name="SamFunc1")] @@ -143,8 +142,8 @@ def test_with_binary_media_types_in_swagger_and_on_resource(self): expected_routes = [Route(path="/path", methods=["OPTIONS"], function_name="SamFunc1")] provider = ApiProvider(template) - assertCountEqual(self, expected_routes, provider.routes) - assertCountEqual(self, provider.api.binary_media_types, expected_binary_types) + self.assertCountEqual(expected_routes, provider.routes) + self.assertCountEqual(provider.api.binary_media_types, expected_binary_types) class TestCloudFormationStageValues(TestCase): @@ -377,8 +376,7 @@ def test_resolve_correct_multi_parent_resource_path(self): } provider = ApiProvider(template) - assertCountEqual( - self, + self.assertCountEqual( provider.routes, [ Route(path="/root/v1/beta", methods=["POST"], function_name=None), @@ -401,8 +399,7 @@ def test_resource_with_method_correct_routes(self): } } provider = ApiProvider(template) - assertCountEqual( - self, + self.assertCountEqual( provider.routes, [ Route( @@ -475,8 +472,7 @@ def test_method_integration_uri(self): } provider = ApiProvider(template) - assertCountEqual( - self, + self.assertCountEqual( provider.routes, [ Route(path="/root/v1/beta", methods=["POST"], function_name="AWSLambdaFunction"), @@ -550,7 +546,7 @@ def test_binary_media_types_method(self): } provider = ApiProvider(template) - assertCountEqual(self, provider.api.binary_media_types, ["image/png", "image/jpg"]) + self.assertCountEqual(provider.api.binary_media_types, ["image/png", "image/jpg"]) def test_cdk(self): template = { @@ -668,4 +664,4 @@ def test_cdk(self): provider = ApiProvider(template) proxy_paths = [Route(path="/{proxy+}", methods=Route.ANY_HTTP_METHODS, function_name="HelloHandler2E4FBA4D")] root_paths = [Route(path="/", methods=Route.ANY_HTTP_METHODS, function_name="HelloHandler2E4FBA4D")] - assertCountEqual(self, provider.routes, proxy_paths + root_paths) + self.assertCountEqual(provider.routes, proxy_paths + root_paths) diff --git a/tests/unit/commands/local/lib/test_local_api_service.py b/tests/unit/commands/local/lib/test_local_api_service.py index 758007c6c5..d695d4ca9d 100644 --- a/tests/unit/commands/local/lib/test_local_api_service.py +++ b/tests/unit/commands/local/lib/test_local_api_service.py @@ -4,7 +4,7 @@ from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from samcli.commands.local.lib.provider import Api from samcli.commands.local.lib.api_collector import ApiCollector diff --git a/tests/unit/commands/local/lib/test_local_lambda.py b/tests/unit/commands/local/lib/test_local_lambda.py index ee9f5ced3f..9a81491980 100644 --- a/tests/unit/commands/local/lib/test_local_lambda.py +++ b/tests/unit/commands/local/lib/test_local_lambda.py @@ -2,7 +2,7 @@ Testing local lambda runner """ from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from parameterized import parameterized, param from samcli.commands.local.lib.local_lambda import LocalLambdaRunner diff --git a/tests/unit/commands/local/lib/test_local_lambda_service.py b/tests/unit/commands/local/lib/test_local_lambda_service.py index f4efd8a148..da28d4711f 100644 --- a/tests/unit/commands/local/lib/test_local_lambda_service.py +++ b/tests/unit/commands/local/lib/test_local_lambda_service.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from samcli.commands.local.lib.local_lambda_service import LocalLambdaService diff --git a/tests/unit/commands/local/lib/test_sam_api_provider.py b/tests/unit/commands/local/lib/test_sam_api_provider.py index 7530dcb6bf..612724d637 100644 --- a/tests/unit/commands/local/lib/test_sam_api_provider.py +++ b/tests/unit/commands/local/lib/test_sam_api_provider.py @@ -3,9 +3,8 @@ from collections import OrderedDict from unittest import TestCase -from mock import patch +from unittest.mock import patch from nose_parameterized import parameterized -from six import assertCountEqual from samcli.commands.validate.lib.exceptions import InvalidSamDocumentException from samcli.commands.local.lib.api_provider import ApiProvider @@ -224,7 +223,7 @@ def test_provider_must_support_binary_media_types(self): self.assertEqual(len(provider.routes), 1) self.assertEqual(list(provider.routes)[0], Route(path="/path", methods=["GET"], function_name="SamFunc1")) - assertCountEqual(self, provider.api.binary_media_types, ["image/gif", "image/png"]) + self.assertCountEqual(provider.api.binary_media_types, ["image/gif", "image/png"]) self.assertEqual(provider.api.stage_name, "Prod") def test_provider_must_support_binary_media_types_with_any_method(self): @@ -255,8 +254,8 @@ def test_provider_must_support_binary_media_types_with_any_method(self): provider = ApiProvider(template) - assertCountEqual(self, provider.routes, expected_routes) - assertCountEqual(self, provider.api.binary_media_types, binary) + self.assertCountEqual(provider.routes, expected_routes) + self.assertCountEqual(provider.api.binary_media_types, binary) class TestSamApiProviderWithExplicitApis(TestCase): @@ -287,7 +286,7 @@ def test_with_inline_swagger_routes(self): } provider = ApiProvider(template) - assertCountEqual(self, self.input_routes, provider.routes) + self.assertCountEqual(self.input_routes, provider.routes) def test_with_swagger_as_local_file(self): with tempfile.NamedTemporaryFile(mode="w", delete=False) as fp: @@ -307,7 +306,7 @@ def test_with_swagger_as_local_file(self): } provider = ApiProvider(template) - assertCountEqual(self, self.input_routes, provider.routes) + self.assertCountEqual(self.input_routes, provider.routes) @patch("samcli.commands.local.lib.cfn_base_api_provider.SwaggerReader") def test_with_swagger_as_both_body_and_uri_called(self, SwaggerReaderMock): @@ -327,7 +326,7 @@ def test_with_swagger_as_both_body_and_uri_called(self, SwaggerReaderMock): cwd = "foo" provider = ApiProvider(template, cwd=cwd) - assertCountEqual(self, self.input_routes, provider.routes) + self.assertCountEqual(self.input_routes, provider.routes) SwaggerReaderMock.assert_called_with(definition_body=body, definition_uri=filename, working_dir=cwd) def test_swagger_with_any_method(self): @@ -351,7 +350,7 @@ def test_swagger_with_any_method(self): } provider = ApiProvider(template) - assertCountEqual(self, expected_routes, provider.routes) + self.assertCountEqual(expected_routes, provider.routes) def test_with_binary_media_types(self): template = { @@ -374,8 +373,8 @@ def test_with_binary_media_types(self): ] provider = ApiProvider(template) - assertCountEqual(self, expected_routes, provider.routes) - assertCountEqual(self, provider.api.binary_media_types, expected_binary_types) + self.assertCountEqual(expected_routes, provider.routes) + self.assertCountEqual(provider.api.binary_media_types, expected_binary_types) def test_with_binary_media_types_in_swagger_and_on_resource(self): input_routes = [Route(path="/path", methods=["OPTIONS"], function_name="SamFunc1")] @@ -398,8 +397,8 @@ def test_with_binary_media_types_in_swagger_and_on_resource(self): expected_routes = [Route(path="/path", methods=["OPTIONS"], function_name="SamFunc1")] provider = ApiProvider(template) - assertCountEqual(self, expected_routes, provider.routes) - assertCountEqual(self, provider.api.binary_media_types, expected_binary_types) + self.assertCountEqual(expected_routes, provider.routes) + self.assertCountEqual(provider.api.binary_media_types, expected_binary_types) class TestSamApiProviderWithExplicitAndImplicitApis(TestCase): @@ -445,7 +444,7 @@ def test_must_union_implicit_and_explicit(self): ] provider = ApiProvider(self.template) - assertCountEqual(self, expected_routes, provider.routes) + self.assertCountEqual(expected_routes, provider.routes) def test_must_prefer_implicit_api_over_explicit(self): implicit_routes = { @@ -473,7 +472,7 @@ def test_must_prefer_implicit_api_over_explicit(self): ] provider = ApiProvider(self.template) - assertCountEqual(self, expected_routes, provider.routes) + self.assertCountEqual(expected_routes, provider.routes) def test_must_prefer_implicit_with_any_method(self): implicit_routes = { @@ -505,7 +504,7 @@ def test_must_prefer_implicit_with_any_method(self): ] provider = ApiProvider(self.template) - assertCountEqual(self, expected_routes, provider.routes) + self.assertCountEqual(expected_routes, provider.routes) def test_with_any_method_on_both(self): implicit_routes = { @@ -547,7 +546,7 @@ def test_with_any_method_on_both(self): ] provider = ApiProvider(self.template) - assertCountEqual(self, expected_routes, provider.routes) + self.assertCountEqual(expected_routes, provider.routes) def test_must_add_explicit_api_when_ref_with_rest_api_id(self): events = { @@ -583,7 +582,7 @@ def test_must_add_explicit_api_when_ref_with_rest_api_id(self): ] provider = ApiProvider(self.template) - assertCountEqual(self, expected_routes, provider.routes) + self.assertCountEqual(expected_routes, provider.routes) def test_both_routes_must_get_binary_media_types(self): events = { @@ -613,8 +612,8 @@ def test_both_routes_must_get_binary_media_types(self): ] provider = ApiProvider(self.template) - assertCountEqual(self, expected_routes, provider.routes) - assertCountEqual(self, provider.api.binary_media_types, expected_explicit_binary_types) + self.assertCountEqual(expected_routes, provider.routes) + self.assertCountEqual(provider.api.binary_media_types, expected_explicit_binary_types) def test_binary_media_types_with_rest_api_id_reference(self): events = { @@ -650,8 +649,8 @@ def test_binary_media_types_with_rest_api_id_reference(self): ] provider = ApiProvider(self.template) - assertCountEqual(self, expected_routes, provider.routes) - assertCountEqual(self, provider.api.binary_media_types, expected_explicit_binary_types) + self.assertCountEqual(expected_routes, provider.routes) + self.assertCountEqual(provider.api.binary_media_types, expected_explicit_binary_types) class TestSamStageValues(TestCase): diff --git a/tests/unit/commands/local/lib/test_sam_base_provider.py b/tests/unit/commands/local/lib/test_sam_base_provider.py index 0eebe68522..2123aac4c8 100644 --- a/tests/unit/commands/local/lib/test_sam_base_provider.py +++ b/tests/unit/commands/local/lib/test_sam_base_provider.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from samcli.commands.local.lib.sam_base_provider import SamBaseProvider from samcli.lib.intrinsic_resolver.intrinsic_property_resolver import IntrinsicResolver diff --git a/tests/unit/commands/local/lib/test_sam_function_provider.py b/tests/unit/commands/local/lib/test_sam_function_provider.py index 6973356e0d..9b8e679159 100644 --- a/tests/unit/commands/local/lib/test_sam_function_provider.py +++ b/tests/unit/commands/local/lib/test_sam_function_provider.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch +from unittest.mock import patch from parameterized import parameterized from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn diff --git a/tests/unit/commands/local/start_api/test_cli.py b/tests/unit/commands/local/start_api/test_cli.py index b9d514e434..e904944437 100644 --- a/tests/unit/commands/local/start_api/test_cli.py +++ b/tests/unit/commands/local/start_api/test_cli.py @@ -3,7 +3,7 @@ """ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from parameterized import parameterized diff --git a/tests/unit/commands/local/start_lambda/test_cli.py b/tests/unit/commands/local/start_lambda/test_cli.py index 91e46689ce..5dff100199 100644 --- a/tests/unit/commands/local/start_lambda/test_cli.py +++ b/tests/unit/commands/local/start_lambda/test_cli.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from parameterized import parameterized diff --git a/tests/unit/commands/logs/test_command.py b/tests/unit/commands/logs/test_command.py index a1938a141c..b895428f19 100644 --- a/tests/unit/commands/logs/test_command.py +++ b/tests/unit/commands/logs/test_command.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch, call +from unittest.mock import Mock, patch, call from samcli.commands.logs.command import do_cli diff --git a/tests/unit/commands/logs/test_logs_context.py b/tests/unit/commands/logs/test_logs_context.py index 630a1413dd..fe37d4e1c7 100644 --- a/tests/unit/commands/logs/test_logs_context.py +++ b/tests/unit/commands/logs/test_logs_context.py @@ -2,7 +2,7 @@ from botocore.stub import Stubber from unittest import TestCase -from mock import Mock, patch, ANY +from unittest.mock import Mock, patch, ANY from samcli.commands.logs.logs_context import LogsCommandContext from samcli.commands.exceptions import UserException diff --git a/tests/unit/commands/publish/test_command.py b/tests/unit/commands/publish/test_command.py index 743ac4e4fc..0225d9f42f 100644 --- a/tests/unit/commands/publish/test_command.py +++ b/tests/unit/commands/publish/test_command.py @@ -1,7 +1,7 @@ """Test sam publish CLI.""" import json from unittest import TestCase -from mock import patch, call, Mock +from unittest.mock import patch, call, Mock from serverlessrepo.exceptions import ServerlessRepoError, InvalidS3UriError from serverlessrepo.publish import CREATE_APPLICATION, UPDATE_APPLICATION diff --git a/tests/unit/commands/test_deploy.py b/tests/unit/commands/test_deploy.py index 119a7ddd2c..90387d5095 100644 --- a/tests/unit/commands/test_deploy.py +++ b/tests/unit/commands/test_deploy.py @@ -3,7 +3,7 @@ """ from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.commands.deploy import do_cli as deploy_cli diff --git a/tests/unit/commands/test_package.py b/tests/unit/commands/test_package.py index a603b10a66..2e368f4d80 100644 --- a/tests/unit/commands/test_package.py +++ b/tests/unit/commands/test_package.py @@ -3,7 +3,7 @@ """ from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.commands.package import do_cli as package_cli diff --git a/tests/unit/commands/validate/lib/test_sam_template_validator.py b/tests/unit/commands/validate/lib/test_sam_template_validator.py index 6a53ee2d87..5a1c0b156a 100644 --- a/tests/unit/commands/validate/lib/test_sam_template_validator.py +++ b/tests/unit/commands/validate/lib/test_sam_template_validator.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from samtranslator.public.exceptions import InvalidDocumentException diff --git a/tests/unit/commands/validate/test_cli.py b/tests/unit/commands/validate/test_cli.py index 156a614619..5f9cac7a9a 100644 --- a/tests/unit/commands/validate/test_cli.py +++ b/tests/unit/commands/validate/test_cli.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from botocore.exceptions import NoCredentialsError diff --git a/tests/unit/lib/build_module/test_app_builder.py b/tests/unit/lib/build_module/test_app_builder.py index cb617ebfa5..fb8e2def92 100644 --- a/tests/unit/lib/build_module/test_app_builder.py +++ b/tests/unit/lib/build_module/test_app_builder.py @@ -3,13 +3,8 @@ import json from unittest import TestCase -from mock import Mock, call, patch - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path - +from unittest.mock import Mock, call, patch +from pathlib import Path from samcli.lib.build.app_builder import ( ApplicationBuilder, diff --git a/tests/unit/lib/build_module/test_workflow_config.py b/tests/unit/lib/build_module/test_workflow_config.py index 14a29fee2f..eab6393b7a 100644 --- a/tests/unit/lib/build_module/test_workflow_config.py +++ b/tests/unit/lib/build_module/test_workflow_config.py @@ -1,6 +1,6 @@ from unittest import TestCase from parameterized import parameterized -from mock import patch +from unittest.mock import patch from samcli.lib.build.workflow_config import get_workflow_config, UnsupportedRuntimeException diff --git a/tests/unit/lib/intrinsic_resolver/test_intrinsic_resolver.py b/tests/unit/lib/intrinsic_resolver/test_intrinsic_resolver.py index 0c80e84eab..047a292a63 100644 --- a/tests/unit/lib/intrinsic_resolver/test_intrinsic_resolver.py +++ b/tests/unit/lib/intrinsic_resolver/test_intrinsic_resolver.py @@ -1,14 +1,9 @@ import json from collections import OrderedDict from copy import deepcopy - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path from unittest import TestCase - -from mock import patch +from unittest.mock import patch from parameterized import parameterized diff --git a/tests/unit/lib/intrinsic_resolver/test_intrinsics_symbol_table.py b/tests/unit/lib/intrinsic_resolver/test_intrinsics_symbol_table.py index 6d03cb16ed..49a7147b6e 100644 --- a/tests/unit/lib/intrinsic_resolver/test_intrinsics_symbol_table.py +++ b/tests/unit/lib/intrinsic_resolver/test_intrinsics_symbol_table.py @@ -1,6 +1,6 @@ from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.lib.intrinsic_resolver.invalid_intrinsic_exception import InvalidSymbolException from samcli.lib.intrinsic_resolver.intrinsic_property_resolver import IntrinsicResolver diff --git a/tests/unit/lib/logs/test_fetcher.py b/tests/unit/lib/logs/test_fetcher.py index 96fb69d2b5..c0b634c008 100644 --- a/tests/unit/lib/logs/test_fetcher.py +++ b/tests/unit/lib/logs/test_fetcher.py @@ -3,7 +3,7 @@ import botocore.session from unittest import TestCase -from mock import Mock, patch, call, ANY +from unittest.mock import Mock, patch, call, ANY from botocore.stub import Stubber from samcli.lib.logs.fetcher import LogsFetcher diff --git a/tests/unit/lib/logs/test_formatter.py b/tests/unit/lib/logs/test_formatter.py index 5fe1828b59..79f5e85a35 100644 --- a/tests/unit/lib/logs/test_formatter.py +++ b/tests/unit/lib/logs/test_formatter.py @@ -1,7 +1,7 @@ import json from unittest import TestCase -from mock import Mock, patch, call +from unittest.mock import Mock, patch, call from nose_parameterized import parameterized from samcli.lib.logs.formatter import LogsFormatter, LambdaLogMsgFormatters, KeywordHighlighter, JSONMsgFormatter diff --git a/tests/unit/lib/samlib/test_cloudformation_command.py b/tests/unit/lib/samlib/test_cloudformation_command.py index d5ce254ebc..e846570c96 100644 --- a/tests/unit/lib/samlib/test_cloudformation_command.py +++ b/tests/unit/lib/samlib/test_cloudformation_command.py @@ -6,7 +6,7 @@ from subprocess import CalledProcessError, PIPE from unittest import TestCase -from mock import patch, call, ANY +from unittest.mock import patch, call, ANY from samcli.lib.samlib.cloudformation_command import execute_command, find_executable diff --git a/tests/unit/lib/telemetry/test_metrics.py b/tests/unit/lib/telemetry/test_metrics.py index 3ea872014c..1ca602512f 100644 --- a/tests/unit/lib/telemetry/test_metrics.py +++ b/tests/unit/lib/telemetry/test_metrics.py @@ -2,7 +2,7 @@ import time from unittest import TestCase -from mock import patch, Mock, ANY, call +from unittest.mock import patch, Mock, ANY, call from samcli.lib.telemetry.metrics import send_installed_metric, track_command from samcli.commands.exceptions import UserException diff --git a/tests/unit/lib/telemetry/test_telemetry.py b/tests/unit/lib/telemetry/test_telemetry.py index 9d57eafde0..e9ab045508 100644 --- a/tests/unit/lib/telemetry/test_telemetry.py +++ b/tests/unit/lib/telemetry/test_telemetry.py @@ -1,7 +1,7 @@ import platform import requests -from mock import patch, Mock, ANY +from unittest.mock import patch, Mock, ANY from unittest import TestCase from samcli.lib.telemetry.telemetry import Telemetry diff --git a/tests/unit/lib/utils/test_codeuri.py b/tests/unit/lib/utils/test_codeuri.py index 45052da423..c5682bbe8d 100644 --- a/tests/unit/lib/utils/test_codeuri.py +++ b/tests/unit/lib/utils/test_codeuri.py @@ -2,10 +2,7 @@ from unittest import TestCase from parameterized import parameterized -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path from samcli.lib.utils.codeuri import resolve_code_path diff --git a/tests/unit/lib/utils/test_progressbar.py b/tests/unit/lib/utils/test_progressbar.py index d082d770d8..6305ea12da 100644 --- a/tests/unit/lib/utils/test_progressbar.py +++ b/tests/unit/lib/utils/test_progressbar.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from samcli.lib.utils.progressbar import progressbar diff --git a/tests/unit/lib/utils/test_sam_logging.py b/tests/unit/lib/utils/test_sam_logging.py index 44fb2a63b2..b2fb1654ce 100644 --- a/tests/unit/lib/utils/test_sam_logging.py +++ b/tests/unit/lib/utils/test_sam_logging.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from samcli.lib.utils.sam_logging import SamCliLogger diff --git a/tests/unit/lib/utils/test_stream_writer.py b/tests/unit/lib/utils/test_stream_writer.py index d882857488..cb48955850 100644 --- a/tests/unit/lib/utils/test_stream_writer.py +++ b/tests/unit/lib/utils/test_stream_writer.py @@ -6,7 +6,7 @@ from samcli.lib.utils.stream_writer import StreamWriter -from mock import Mock +from unittest.mock import Mock class TestStreamWriter(TestCase): diff --git a/tests/unit/lib/utils/test_tar.py b/tests/unit/lib/utils/test_tar.py index 2781362106..d5df37240b 100644 --- a/tests/unit/lib/utils/test_tar.py +++ b/tests/unit/lib/utils/test_tar.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch, call +from unittest.mock import Mock, patch, call from samcli.lib.utils.tar import create_tarball diff --git a/tests/unit/local/apigw/test_local_apigw_service.py b/tests/unit/local/apigw/test_local_apigw_service.py index dfb3003c78..bd05a66b4c 100644 --- a/tests/unit/local/apigw/test_local_apigw_service.py +++ b/tests/unit/local/apigw/test_local_apigw_service.py @@ -3,7 +3,7 @@ import json from unittest import TestCase -from mock import Mock, patch, ANY, MagicMock +from unittest.mock import Mock, patch, ANY, MagicMock from parameterized import parameterized, param from werkzeug.datastructures import Headers diff --git a/tests/unit/local/apigw/test_service_error_responses.py b/tests/unit/local/apigw/test_service_error_responses.py index 433ff30241..f42314e0aa 100644 --- a/tests/unit/local/apigw/test_service_error_responses.py +++ b/tests/unit/local/apigw/test_service_error_responses.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from samcli.local.apigw.service_error_responses import ServiceErrorResponses diff --git a/tests/unit/local/docker/test_container.py b/tests/unit/local/docker/test_container.py index ce07dc7abf..e38ff04486 100644 --- a/tests/unit/local/docker/test_container.py +++ b/tests/unit/local/docker/test_container.py @@ -3,7 +3,7 @@ """ from docker.errors import NotFound, APIError from unittest import TestCase -from mock import Mock, call, patch +from unittest.mock import Mock, call, patch from samcli.local.docker.container import Container diff --git a/tests/unit/local/docker/test_lambda_build_container.py b/tests/unit/local/docker/test_lambda_build_container.py index 037d1450d3..8a3da6a30b 100644 --- a/tests/unit/local/docker/test_lambda_build_container.py +++ b/tests/unit/local/docker/test_lambda_build_container.py @@ -3,14 +3,10 @@ """ import json - -try: - import pathlib -except ImportError: - import pathlib2 as pathlib +import pathlib from unittest import TestCase -from mock import patch +from unittest.mock import patch from parameterized import parameterized diff --git a/tests/unit/local/docker/test_lambda_container.py b/tests/unit/local/docker/test_lambda_container.py index cd7afdcd33..30db16c83a 100644 --- a/tests/unit/local/docker/test_lambda_container.py +++ b/tests/unit/local/docker/test_lambda_container.py @@ -3,7 +3,7 @@ """ from unittest import TestCase -from mock import patch, Mock +from unittest.mock import patch, Mock from parameterized import parameterized, param from samcli.commands.local.lib.debug_context import DebugContext diff --git a/tests/unit/local/docker/test_lambda_image.py b/tests/unit/local/docker/test_lambda_image.py index a9d3972c6d..7a194b10c6 100644 --- a/tests/unit/local/docker/test_lambda_image.py +++ b/tests/unit/local/docker/test_lambda_image.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch, Mock, mock_open +from unittest.mock import patch, Mock, mock_open from docker.errors import ImageNotFound, BuildError, APIError diff --git a/tests/unit/local/docker/test_manager.py b/tests/unit/local/docker/test_manager.py index 835631300f..444437e153 100644 --- a/tests/unit/local/docker/test_manager.py +++ b/tests/unit/local/docker/test_manager.py @@ -7,7 +7,7 @@ import requests -from mock import Mock +from unittest.mock import Mock from docker.errors import APIError, ImageNotFound from samcli.local.docker.manager import ContainerManager, DockerImagePullFailedException diff --git a/tests/unit/local/docker/test_utils.py b/tests/unit/local/docker/test_utils.py index 2e4af7ca02..09ab511e55 100644 --- a/tests/unit/local/docker/test_utils.py +++ b/tests/unit/local/docker/test_utils.py @@ -4,8 +4,7 @@ import os from unittest import TestCase - -from mock import patch +from unittest.mock import patch from samcli.local.docker.utils import to_posix_path diff --git a/tests/unit/local/events/test_api_event.py b/tests/unit/local/events/test_api_event.py index c2c425dab9..aa106f5ebd 100644 --- a/tests/unit/local/events/test_api_event.py +++ b/tests/unit/local/events/test_api_event.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from samcli.local.events.api_event import ContextIdentity, RequestContext, ApiGatewayLambdaEvent diff --git a/tests/unit/local/init/test_init.py b/tests/unit/local/init/test_init.py index b4d29c3979..a799640e94 100644 --- a/tests/unit/local/init/test_init.py +++ b/tests/unit/local/init/test_init.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch +from unittest.mock import patch from cookiecutter.exceptions import CookiecutterException from samcli.local.init import generate_project diff --git a/tests/unit/local/lambda_service/test_lambda_error_responses.py b/tests/unit/local/lambda_service/test_lambda_error_responses.py index 699e70371f..47190f0ea4 100644 --- a/tests/unit/local/lambda_service/test_lambda_error_responses.py +++ b/tests/unit/local/lambda_service/test_lambda_error_responses.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import patch +from unittest.mock import patch from samcli.local.lambda_service.lambda_error_responses import LambdaErrorResponses diff --git a/tests/unit/local/lambda_service/test_local_lambda_invoke_service.py b/tests/unit/local/lambda_service/test_local_lambda_invoke_service.py index ae92926c4c..14efa11e5c 100644 --- a/tests/unit/local/lambda_service/test_local_lambda_invoke_service.py +++ b/tests/unit/local/lambda_service/test_local_lambda_invoke_service.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch, ANY, call +from unittest.mock import Mock, patch, ANY, call from samcli.local.lambda_service.local_lambda_invoke_service import LocalLambdaInvokeService from samcli.local.lambdafn.exceptions import FunctionNotFound diff --git a/tests/unit/local/lambdafn/test_config.py b/tests/unit/local/lambdafn/test_config.py index d53e7d3ee3..07b240459b 100644 --- a/tests/unit/local/lambdafn/test_config.py +++ b/tests/unit/local/lambdafn/test_config.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock +from unittest.mock import Mock from samcli.local.lambdafn.config import FunctionConfig diff --git a/tests/unit/local/lambdafn/test_env_vars.py b/tests/unit/local/lambdafn/test_env_vars.py index 3e319fc682..5491df9b87 100644 --- a/tests/unit/local/lambdafn/test_env_vars.py +++ b/tests/unit/local/lambdafn/test_env_vars.py @@ -305,7 +305,7 @@ def test_must_replace_non_scalar_with_blank_values(self, input): (False, "false"), (1234, "1234"), (3.14, "3.14"), - (u"mystring\xe0", u"mystring\xe0"), + ("mystring\xe0", "mystring\xe0"), ("mystring", "mystring"), ] ) diff --git a/tests/unit/local/lambdafn/test_runtime.py b/tests/unit/local/lambdafn/test_runtime.py index 688219cd61..74e585b42c 100644 --- a/tests/unit/local/lambdafn/test_runtime.py +++ b/tests/unit/local/lambdafn/test_runtime.py @@ -3,7 +3,7 @@ """ from unittest import TestCase -from mock import Mock, patch, MagicMock, ANY +from unittest.mock import Mock, patch, MagicMock, ANY from parameterized import parameterized from samcli.local.lambdafn.runtime import LambdaRuntime, _unzip_file diff --git a/tests/unit/local/lambdafn/test_zip.py b/tests/unit/local/lambdafn/test_zip.py index 5f36b6023f..ade9cd8da0 100644 --- a/tests/unit/local/lambdafn/test_zip.py +++ b/tests/unit/local/lambdafn/test_zip.py @@ -8,7 +8,7 @@ from unittest import TestCase from unittest import skipIf -from mock import Mock, patch +from unittest.mock import Mock, patch from nose_parameterized import parameterized, param from samcli.local.lambdafn.zip import unzip, unzip_from_uri, _override_permissions diff --git a/tests/unit/local/layers/test_download_layers.py b/tests/unit/local/layers/test_download_layers.py index 288252bfbe..3c49f34207 100644 --- a/tests/unit/local/layers/test_download_layers.py +++ b/tests/unit/local/layers/test_download_layers.py @@ -1,12 +1,8 @@ from unittest import TestCase -from mock import patch, Mock, call +from unittest.mock import Mock, call, patch from botocore.exceptions import NoCredentialsError, ClientError - -try: - from pathlib import Path -except ImportError: - from pathlib2 import Path +from pathlib import Path from samcli.local.layers.layer_downloader import LayerDownloader diff --git a/tests/unit/local/services/test_base_local_service.py b/tests/unit/local/services/test_base_local_service.py index adb53989fc..8ba46c7aa2 100644 --- a/tests/unit/local/services/test_base_local_service.py +++ b/tests/unit/local/services/test_base_local_service.py @@ -1,5 +1,5 @@ from unittest import TestCase -from mock import Mock, patch +from unittest.mock import Mock, patch from parameterized import parameterized, param From ccefe3494a7e51dfbeb3b0b1e392a1253c722e07 Mon Sep 17 00:00:00 2001 From: Tina-Wang-0904 <56325110+Tina-Wang-0904@users.noreply.github.com> Date: Wed, 23 Oct 2019 16:40:11 -0700 Subject: [PATCH 24/49] fix: typo in CloudFrontServeObjectOnViewer event (#1451) --- .../events/cloudfront/CloudFrontServeObjectOnViewerDevice.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/local/lib/generated_sample_events/events/cloudfront/CloudFrontServeObjectOnViewerDevice.json b/samcli/commands/local/lib/generated_sample_events/events/cloudfront/CloudFrontServeObjectOnViewerDevice.json index 3c299d57ca..8bbe5dae90 100644 --- a/samcli/commands/local/lib/generated_sample_events/events/cloudfront/CloudFrontServeObjectOnViewerDevice.json +++ b/samcli/commands/local/lib/generated_sample_events/events/cloudfront/CloudFrontServeObjectOnViewerDevice.json @@ -16,7 +16,7 @@ "value": "d123.cf.net" } ], - "cloudfront-is-dektop-viewer": [ + "cloudfront-is-desktop-viewer": [ { "key": "CloudFront-Is-Desktop-Viewer", "value": "true" From 458668bafad22219fd2e1e9da2fa267035b7a9f1 Mon Sep 17 00:00:00 2001 From: Jacob Fuss <32497805+jfuss@users.noreply.github.com> Date: Thu, 24 Oct 2019 15:03:31 -0500 Subject: [PATCH 25/49] chore: add some functional tests to appveyor (#1450) * chore: add some functional tests to appveyor * print python version * Change JSONDecodeError to ValueError * Remove the extra python --version * fix functional global config tests * adjust functional tests that run * don't run manager functional tests * fix functional tests that are run in appveyor * Mark invoke and start-lambda integ tests as flaky This is due to random errors/timeouts happening on windows. The start-api tests already have this flaky annotation but invoke and start-lambda were missed --- Makefile | 2 +- appveyor.yml | 6 ++++++ .../commands/cli/test_global_config.py | 9 +++++--- .../local/docker/test_container_manager.py | 2 +- .../local/docker/test_lambda_container.py | 4 ++-- .../functional/local/lambdafn/test_runtime.py | 2 +- .../local/invoke/test_integrations_cli.py | 21 +++++++++++++++++++ .../local/start_lambda/test_start_lambda.py | 10 +++++++++ 8 files changed, 48 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index b9084260bf..a59f87b19d 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ integ-test: func-test: # Verify function test coverage only for `samcli.local` package @echo Telemetry Status: $(SAM_CLI_TELEMETRY) - pytest --cov samcli.local --cov samcli.commands.local --cov-report term-missing tests/functional + pytest --cov samcli.local --cov samcli.commands.local --cov-report term-missing tests/functional/commands/validate tests/functional/commands/cli/test_global_config.py smoke-test: # Smoke tests run in parallel diff --git a/appveyor.yml b/appveyor.yml index c7bf8d775c..03aa53763d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -54,6 +54,9 @@ for: - "venv\\Scripts\\activate" - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit" - "pylint --rcfile .pylintrc samcli" + # There are some functional tests that are currently broken due to not being updated with changed code or still running with node4.3 runtimes + # We need to update those but this allows us to at least runs the ones we currently have working + - "pytest tests/functional/commands/validate tests/functional/commands/cli/test_global_config.py" # Runs only in Linux - sh: "pytest -vv tests/integration" @@ -94,6 +97,9 @@ for: test_script: - "pytest --cov samcli --cov-report term-missing --cov-fail-under 95 tests/unit" - "pylint --rcfile .pylintrc samcli" + # There are some functional tests that are currently broken due to not being updated with changed code or still running with node4.3 runtimes + # We need to update those but this allows us to at least runs the ones we currently have working + - "pytest tests/functional/commands/validate tests/functional/commands/cli/test_global_config.py" # Runs only in Linux - sh: "pytest -vv tests/integration" diff --git a/tests/functional/commands/cli/test_global_config.py b/tests/functional/commands/cli/test_global_config.py index ed550f762a..aa091443b0 100644 --- a/tests/functional/commands/cli/test_global_config.py +++ b/tests/functional/commands/cli/test_global_config.py @@ -1,10 +1,10 @@ import json import tempfile import shutil +import os from unittest.mock import mock_open, patch from unittest import TestCase -from json import JSONDecodeError from samcli.cli.global_config import GlobalConfig from pathlib import Path @@ -12,9 +12,12 @@ class TestGlobalConfig(TestCase): def setUp(self): self._cfg_dir = tempfile.mkdtemp() + self._previous_telemetry_environ = os.environ.get("SAM_CLI_TELEMETRY") + os.environ.pop("SAM_CLI_TELEMETRY") def tearDown(self): shutil.rmtree(self._cfg_dir) + os.environ["SAM_CLI_TELEMETRY"] = self._previous_telemetry_environ def test_installation_id_with_side_effect(self): gc = GlobalConfig(config_dir=self._cfg_dir) @@ -87,7 +90,7 @@ def test_telemetry_flag_not_in_cfg(self): def test_set_telemetry_flag_no_file(self): path = Path(self._cfg_dir, "metadata.json") gc = GlobalConfig(config_dir=self._cfg_dir) - self.assertIsNone(gc.telemetry_enabled) # pre-state test + self.assertFalse(gc.telemetry_enabled) # pre-state test gc.telemetry_enabled = True from_gc = gc.telemetry_enabled json_body = json.loads(path.read_text()) @@ -131,7 +134,7 @@ def test_setter_raises_on_invalid_json(self): with open(str(path), "w") as f: f.write("NOT JSON, PROBABLY VALID YAML AM I RIGHT!?") gc = GlobalConfig(config_dir=self._cfg_dir) - with self.assertRaises(JSONDecodeError): + with self.assertRaises(ValueError): gc.telemetry_enabled = True def test_setter_cannot_open_file(self): diff --git a/tests/functional/local/docker/test_container_manager.py b/tests/functional/local/docker/test_container_manager.py index 1dbbc2c051..01c15d8c5a 100644 --- a/tests/functional/local/docker/test_container_manager.py +++ b/tests/functional/local/docker/test_container_manager.py @@ -9,7 +9,7 @@ class TestContainerManager(TestCase): Verifies functionality of ContainerManager by calling Docker APIs """ - IMAGE = "busybox" # small sized Linux container + IMAGE = "busybox:latest" # small sized Linux container @classmethod def setUpClass(cls): diff --git a/tests/functional/local/docker/test_lambda_container.py b/tests/functional/local/docker/test_lambda_container.py index 28326f6a1c..178487d142 100644 --- a/tests/functional/local/docker/test_lambda_container.py +++ b/tests/functional/local/docker/test_lambda_container.py @@ -27,7 +27,7 @@ class TestLambdaContainer(TestCase): necessary to tests them here. """ - IMAGE_NAME = "lambci/lambda:nodejs4.3" + IMAGE_NAME = "lambci/lambda:nodejs10.x" HELLO_WORLD_CODE = """ exports.handler = function(event, context, callback){ @@ -47,7 +47,7 @@ def setUpClass(cls): def setUp(self): random.seed() - self.runtime = "nodejs4.3" + self.runtime = "nodejs10.x" self.expected_docker_image = self.IMAGE_NAME self.handler = "index.handler" self.layers = [] diff --git a/tests/functional/local/lambdafn/test_runtime.py b/tests/functional/local/lambdafn/test_runtime.py index 4f0ce0152d..2f9e34a4b5 100644 --- a/tests/functional/local/lambdafn/test_runtime.py +++ b/tests/functional/local/lambdafn/test_runtime.py @@ -20,7 +20,7 @@ logging.basicConfig(level=logging.INFO) -RUNTIME = "nodejs4.3" +RUNTIME = "nodejs10.x" HANDLER = "index.handler" MEMORY = 1024 diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index 203ebb7797..d9999d5670 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -25,6 +25,7 @@ class TestSamPython36HelloWorldIntegration(InvokeIntegBase): template = Path("template.yml") + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_returncode_is_zero(self): command_list = self.get_command_list( @@ -36,6 +37,7 @@ def test_invoke_returncode_is_zero(self): self.assertEqual(return_code, 0) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_function_with_metadata(self): command_list = self.get_command_list("FunctionWithMetadata", template_path=self.template_path, no_event=True) @@ -46,6 +48,7 @@ def test_function_with_metadata(self): self.assertEqual(process_stdout.decode("utf-8"), '"Hello World in a different dir"') + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_returns_execpted_results(self): command_list = self.get_command_list( @@ -57,6 +60,7 @@ def test_invoke_returns_execpted_results(self): process_stdout = b"".join(process.stdout.readlines()).strip() self.assertEqual(process_stdout.decode("utf-8"), '"Hello world"') + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_of_lambda_function(self): command_list = self.get_command_list( @@ -68,6 +72,7 @@ def test_invoke_of_lambda_function(self): process_stdout = b"".join(process.stdout.readlines()).strip() self.assertEqual(process_stdout.decode("utf-8"), '"Hello world"') + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") @parameterized.expand([("TimeoutFunction"), ("TimeoutFunctionWithParameter")]) def test_invoke_with_timeout_set(self, function_name): @@ -95,6 +100,7 @@ def test_invoke_with_timeout_set(self, function_name): msg="The return statement in the LambdaFunction " "should never return leading to an empty string", ) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_env_vars(self): command_list = self.get_command_list( @@ -109,6 +115,7 @@ def test_invoke_with_env_vars(self): process_stdout = b"".join(process.stdout.readlines()).strip() self.assertEqual(process_stdout.decode("utf-8"), '"MyVar"') + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_when_function_writes_stdout(self): command_list = self.get_command_list( @@ -124,6 +131,7 @@ def test_invoke_when_function_writes_stdout(self): self.assertIn("Docker Lambda is writing to stdout", process_stderr.decode("utf-8")) self.assertIn("wrote to stdout", process_stdout.decode("utf-8")) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_when_function_writes_stderr(self): command_list = self.get_command_list( @@ -137,6 +145,7 @@ def test_invoke_when_function_writes_stderr(self): self.assertIn("Docker Lambda is writing to stderr", process_stderr.decode("utf-8")) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_returns_expected_result_when_no_event_given(self): command_list = self.get_command_list("EchoEventFunction", template_path=self.template_path) @@ -148,6 +157,7 @@ def test_invoke_returns_expected_result_when_no_event_given(self): self.assertEqual(return_code, 0) self.assertEqual("{}", process_stdout.decode("utf-8")) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_raises_exception_with_noargs_and_event(self): command_list = self.get_command_list( @@ -161,6 +171,7 @@ def test_invoke_raises_exception_with_noargs_and_event(self): error_output = process_stderr.decode("utf-8") self.assertIn("no_event and event cannot be used together. Please provide only one.", error_output) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_env_using_parameters(self): command_list = self.get_command_list( @@ -188,6 +199,7 @@ def test_invoke_with_env_using_parameters(self): self.assertEqual(environ["Timeout"], "100") self.assertEqual(environ["MyRuntimeVersion"], "v0") + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_env_using_parameters_with_custom_region(self): custom_region = "my-custom-region" @@ -203,6 +215,7 @@ def test_invoke_with_env_using_parameters_with_custom_region(self): self.assertEqual(environ["Region"], custom_region) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_env_with_aws_creds(self): custom_region = "my-custom-region" @@ -232,6 +245,7 @@ def test_invoke_with_env_with_aws_creds(self): self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], secret) self.assertEqual(environ["AWS_SESSION_TOKEN"], session) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_docker_network_of_host(self): command_list = self.get_command_list( @@ -246,6 +260,7 @@ def test_invoke_with_docker_network_of_host(self): self.assertEqual(return_code, 0) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") @skipIf(IS_WINDOWS, "The test hangs on Windows due to trying to attach to a non-existing network") def test_invoke_with_docker_network_of_host_in_env_var(self): @@ -262,6 +277,7 @@ def test_invoke_with_docker_network_of_host_in_env_var(self): self.assertIn('Not Found ("network non-existing-network not found")', process_stderr.decode("utf-8")) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_sam_template_file_env_var_set(self): command_list = self.get_command_list("HelloWorldFunctionInNonDefaultTemplate", event_path=self.event_path) @@ -276,6 +292,7 @@ def test_sam_template_file_env_var_set(self): self.assertEqual(process_stdout.decode("utf-8"), '"Hello world"') + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_skip_pull_image_in_env_var(self): docker.from_env().api.pull("lambci/lambda:python3.6") @@ -302,6 +319,7 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.config_dir, ignore_errors=True) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_existing_env_variables_precedence_over_profiles(self): profile = "default" @@ -337,6 +355,7 @@ def test_existing_env_variables_precedence_over_profiles(self): self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], "priority_secret_key_id") self.assertEqual(environ["AWS_SESSION_TOKEN"], "priority_secret_token") + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_default_profile_with_custom_configs(self): profile = "default" @@ -369,6 +388,7 @@ def test_default_profile_with_custom_configs(self): self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], "shhhhhthisisasecret") self.assertEqual(environ["AWS_SESSION_TOKEN"], "sessiontoken") + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_custom_profile_with_custom_configs(self): custom_config = self._create_config_file("custom") @@ -400,6 +420,7 @@ def test_custom_profile_with_custom_configs(self): self.assertEqual(environ["AWS_SECRET_ACCESS_KEY"], "shhhhhthisisasecret") self.assertEqual(environ["AWS_SESSION_TOKEN"], "sessiontoken") + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_custom_profile_through_envrionment_variables(self): # When using a custom profile in a custom location, you need both the config diff --git a/tests/integration/local/start_lambda/test_start_lambda.py b/tests/integration/local/start_lambda/test_start_lambda.py index d2cee30f90..47b9cf742a 100644 --- a/tests/integration/local/start_lambda/test_start_lambda.py +++ b/tests/integration/local/start_lambda/test_start_lambda.py @@ -25,6 +25,7 @@ def setUp(self): config=Config(signature_version=UNSIGNED, read_timeout=120, retries={"max_attempts": 0}), ) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_same_endpoint(self): """ @@ -64,6 +65,7 @@ def setUp(self): config=Config(signature_version=UNSIGNED, read_timeout=120, retries={"max_attempts": 0}), ) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_non_json_data(self): expected_error_message = ( @@ -76,6 +78,7 @@ def test_invoke_with_non_json_data(self): self.assertEqual(str(error.exception), expected_error_message) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_log_type_not_None(self): expected_error_message = ( @@ -88,6 +91,7 @@ def test_invoke_with_log_type_not_None(self): self.assertEqual(str(error.exception), expected_error_message) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_invocation_type_not_RequestResponse(self): expected_error_message = ( @@ -115,6 +119,7 @@ def setUp(self): config=Config(signature_version=UNSIGNED, read_timeout=120, retries={"max_attempts": 0}), ) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_data(self): response = self.lambda_client.invoke(FunctionName="EchoEventFunction", Payload='"This is json data"') @@ -123,6 +128,7 @@ def test_invoke_with_data(self): self.assertIsNone(response.get("FunctionError")) self.assertEqual(response.get("StatusCode"), 200) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_no_data(self): response = self.lambda_client.invoke(FunctionName="EchoEventFunction") @@ -131,6 +137,7 @@ def test_invoke_with_no_data(self): self.assertIsNone(response.get("FunctionError")) self.assertEqual(response.get("StatusCode"), 200) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_log_type_None(self): response = self.lambda_client.invoke(FunctionName="EchoEventFunction", LogType="None") @@ -139,6 +146,7 @@ def test_invoke_with_log_type_None(self): self.assertIsNone(response.get("FunctionError")) self.assertEqual(response.get("StatusCode"), 200) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_invocation_type_RequestResponse(self): response = self.lambda_client.invoke(FunctionName="EchoEventFunction", InvocationType="RequestResponse") @@ -147,6 +155,7 @@ def test_invoke_with_invocation_type_RequestResponse(self): self.assertIsNone(response.get("FunctionError")) self.assertEqual(response.get("StatusCode"), 200) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_lambda_function_raised_error(self): response = self.lambda_client.invoke(FunctionName="RaiseExceptionFunction", InvocationType="RequestResponse") @@ -161,6 +170,7 @@ def test_lambda_function_raised_error(self): self.assertEqual(response.get("FunctionError"), "Unhandled") self.assertEqual(response.get("StatusCode"), 200) + @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") def test_invoke_with_function_timeout(self): """ From b1b45856b907368a4be81a91c6e9df1e89de74d1 Mon Sep 17 00:00:00 2001 From: Trenton Lipscomb Date: Fri, 25 Oct 2019 06:31:54 +0900 Subject: [PATCH 26/49] Matches event params to API params. Response to comments in https://github.com/awslabs/aws-sam-cli/pull/1447 --- .../event-mapping.json | 21 ++++++++----------- .../sagemaker/AnnotationConsolidation.json | 2 +- .../events/sagemaker/PreHumanTask.json | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/samcli/commands/local/lib/generated_sample_events/event-mapping.json b/samcli/commands/local/lib/generated_sample_events/event-mapping.json index 60a6892b9a..fc50fd4010 100644 --- a/samcli/commands/local/lib/generated_sample_events/event-mapping.json +++ b/samcli/commands/local/lib/generated_sample_events/event-mapping.json @@ -863,13 +863,10 @@ "account-id": { "default": "123456789012" }, - "bucket": { - "default": "sagemakerexample" + "source-ref": { + "default": "s3://sagemakerexample/object_to_annotate.jpg" }, - "data_object_key": { - "default": "object_to_annotate.jpg" - }, - "labeling_job_name": { + "labeling-job-name": { "default": "example-job" } } @@ -887,19 +884,19 @@ "account-id": { "default": "123456789012" }, - "labeling_job_name": { + "labeling-job-name": { "default": "example-job" }, - "label_attribute_name": { + "label-attribute-name": { "default": "example-attribute" }, - "bucket": { - "default": "sagemakerexample" + "s3-output-path": { + "default": "s3://sagemakerexample/output" }, - "execution_role": { + "execution-role": { "default": "sagemaker-role" }, - "iteration_object_timestamp": { + "iteration-object-timestamp": { "default": "iteration-1/0/2019-09-06_18:35:03" } } diff --git a/samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json index 083bc76583..7ec6186c70 100644 --- a/samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json +++ b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/AnnotationConsolidation.json @@ -4,6 +4,6 @@ "labelAttributeName": "{{{label_attribute_name}}}", "roleArn" : "aws:{{{partition}}}:iam::{{account_id}}:role/{{{execution_role}}}", "payload": { - "s3Uri": "s3://{{{bucket}}}/{{{labeling_job_name}}}/annotations/worker_response/{{{iteration_object_timestamp}}}.json" + "s3Uri": "{{{s3_output_path}}}/{{{labeling_job_name}}}/annotations/worker_response/{{{iteration_object_timestamp}}}.json" } } diff --git a/samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json index 079e04435c..c7dcae6664 100644 --- a/samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json +++ b/samcli/commands/local/lib/generated_sample_events/events/sagemaker/PreHumanTask.json @@ -2,6 +2,6 @@ "version": "2018-10-16", "labelingJobArn": "arn:{{{partition}}}:sagemaker:{{{region}}}:{{{account_id}}}:labeling-job/{{{labeling_job_name}}}", "dataObject" : { - "source-ref": "s3://{{{bucket}}}/{{{data_object_key}}}" + "source-ref": "{{{source_ref}}}" } } From 8d82ec005f2daa1424753e219f95cc87b5d77bd0 Mon Sep 17 00:00:00 2001 From: Sergey Date: Fri, 25 Oct 2019 16:03:25 +0300 Subject: [PATCH 27/49] feat: Expose multiple ports when running in debug mode (#1479) Issue #1463 --- designs/debug_mode_multiple_exposed_ports.md | 121 ++++++++++++++++++ .../local/cli_common/invoke_context.py | 20 +-- samcli/commands/local/cli_common/options.py | 2 + samcli/commands/local/invoke/cli.py | 2 +- samcli/commands/local/lib/debug_context.py | 13 +- samcli/commands/local/lib/local_lambda.py | 7 +- samcli/commands/local/start_api/cli.py | 2 +- samcli/commands/local/start_lambda/cli.py | 2 +- samcli/local/docker/lambda_container.py | 30 +++-- .../local/docker/test_lambda_container.py | 4 +- .../local/cli_common/test_invoke_context.py | 70 ++++++++-- tests/unit/commands/local/invoke/test_cli.py | 18 +-- .../commands/local/lib/test_debug_context.py | 28 ++-- .../unit/commands/local/start_api/test_cli.py | 6 +- .../commands/local/start_lambda/test_cli.py | 6 +- .../local/docker/test_lambda_container.py | 46 +++++-- 16 files changed, 291 insertions(+), 86 deletions(-) create mode 100644 designs/debug_mode_multiple_exposed_ports.md diff --git a/designs/debug_mode_multiple_exposed_ports.md b/designs/debug_mode_multiple_exposed_ports.md new file mode 100644 index 0000000000..c6486b3925 --- /dev/null +++ b/designs/debug_mode_multiple_exposed_ports.md @@ -0,0 +1,121 @@ +Title: Template for design documents +==================================== + +What is the problem? +-------------------- +Currently, there is one port is exposed from Docker instance when running lambda in debug mode. +This port is used to connect a debugger. In my case, I need two ports to be exposed due to Debugger +implementation specific (the Debugger connect to two sockets to collect different information). + +What will be changed? +--------------------- +SAM CLI has a ``--debug-port`` parameter that provide a port. This parameter is stored in DebugContext object. +``DebugContext`` should store an array of ports instead of a single port. This array should be transformed +into a map containing each stored port when passing to docker container arguments. + +Success criteria for the change +------------------------------- +All ports specified via single or multiple ``--debug-port`` SAM CLI options should be exposed by docker container. + +Out-of-Scope +------------ + +User Experience Walkthrough +--------------------------- +From the user perspective, it should only provide an ability to specify multiple ``--debug-port`` options: +``--debug-port 5600 --debug-port 5601`` + +Implementation +============== + +CLI Changes +----------- + +SAM CLI provide an option to specify multiple ports ``--debug-port 5600 --debug-port 5601``. + +### Breaking Change + +No changes. + +Design +------ + +Update ``--debug-port`` option to allow to use it multiple times in SAM CLI. +The option type should take only integer values. The value is stored in ``DebugContext``. +This value should be converted into a map of ``{ container_port : host_port }`` +that is passed to ``ports`` argument when creating a docker container. + +`.samrc` Changes +---------------- + +No changes. + +Security +-------- + +No changes. + +**What new dependencies (libraries/cli) does this change require?** + +**What other Docker container images are you using?** + +**Are you creating a new HTTP endpoint? If so explain how it will be +created & used** + +**Are you connecting to a remote API? If so explain how is this +connection secured** + +**Are you reading/writing to a temporary folder? If so, what is this +used for and when do you clean up?** + +**How do you validate new .samrc configuration?** + +What is your Testing Plan (QA)? +=============================== + +Goal +---- +Make sure SAM CLI users can specify multiple ports and those ports are exposed +after creating a docker container in debug mode: + +``sam local invoke --template /template.yaml --event /event.json --debugger-path --debug-port 5600 --debug-port 5601`` + +Pre-requesites +-------------- +Running SAM CLI with debug mode. + +Test Scenarios/Cases +-------------------- +1. Single port is specified: ``--debug-port 5600`` +2. Multiple ports are specified: ``--debug-port 5600 --debug-port 5601`` +3. No ports specified: ``--debug-port `` +4. No ``--debug-port`` parameter is specified + +Expected Results +---------------- +1. Single port is exposed in docker container +2. All specified ports are exposed in docker container +3. No ports exposed. +4. No ports exposed. + +Pass/Fail +--------- + +Documentation Changes +===================== + +Open Issues +============ +- [1463](https://github.com/awslabs/aws-sam-cli/issues/1463) + +Task Breakdown +============== + +- \[x\] Send a Pull Request with this design document +- \[ \] Build the command line interface +- \[ \] Build the underlying library +- \[x\] Unit tests +- \[x\] Functional Tests +- \[x\] Integration tests +- \[ \] Run all tests on Windows +- \[x\] Update documentation diff --git a/samcli/commands/local/cli_common/invoke_context.py b/samcli/commands/local/cli_common/invoke_context.py index 6fdbb0584e..cf7821146e 100644 --- a/samcli/commands/local/cli_common/invoke_context.py +++ b/samcli/commands/local/cli_common/invoke_context.py @@ -45,7 +45,7 @@ def __init__( docker_network=None, log_file=None, skip_pull_image=None, - debug_port=None, + debug_ports=None, debug_args=None, debugger_path=None, parameter_overrides=None, @@ -76,8 +76,8 @@ def __init__( Should we skip pulling the Docker container image? aws_profile str Name of the profile to fetch AWS credentials from - debug_port int - Port to bind the debugger to + debug_ports tuple(int) + Ports to bind the debugger to debug_args str Additional arguments passed to the debugger debugger_path str @@ -98,7 +98,7 @@ def __init__( self._docker_network = docker_network self._log_file = log_file self._skip_pull_image = skip_pull_image - self._debug_port = debug_port + self._debug_ports = debug_ports self._debug_args = debug_args self._debugger_path = debugger_path self._parameter_overrides = parameter_overrides or {} @@ -129,7 +129,7 @@ def __enter__(self): self._env_vars_value = self._get_env_vars_value(self._env_vars_file) self._log_file_handle = self._setup_log_file(self._log_file) - self._debug_context = self._get_debug_context(self._debug_port, self._debug_args, self._debugger_path) + self._debug_context = self._get_debug_context(self._debug_ports, self._debug_args, self._debugger_path) self._container_manager = self._get_container_manager(self._docker_network, self._skip_pull_image) @@ -314,14 +314,14 @@ def _setup_log_file(log_file): return open(log_file, "wb") @staticmethod - def _get_debug_context(debug_port, debug_args, debugger_path): + def _get_debug_context(debug_ports, debug_args, debugger_path): """ Creates a DebugContext if the InvokeContext is in a debugging mode Parameters ---------- - debug_port int - Port to bind the debugger to + debug_ports tuple(int) + Ports to bind the debugger to debug_args str Additional arguments passed to the debugger debugger_path str @@ -337,7 +337,7 @@ def _get_debug_context(debug_port, debug_args, debugger_path): samcli.commands.local.cli_common.user_exceptions.DebugContext When the debugger_path is not valid """ - if debug_port and debugger_path: + if debug_ports and debugger_path: try: debugger = Path(debugger_path).resolve(strict=True) except OSError as error: @@ -350,7 +350,7 @@ def _get_debug_context(debug_port, debug_args, debugger_path): raise DebugContextException("'{}' should be a directory with the debugger in it.".format(debugger_path)) debugger_path = str(debugger) - return DebugContext(debug_port=debug_port, debug_args=debug_args, debugger_path=debugger_path) + return DebugContext(debug_ports=debug_ports, debug_args=debug_args, debugger_path=debugger_path) @staticmethod def _get_container_manager(docker_network, skip_pull_image): diff --git a/samcli/commands/local/cli_common/options.py b/samcli/commands/local/cli_common/options.py index d98633a418..af36a1d963 100644 --- a/samcli/commands/local/cli_common/options.py +++ b/samcli/commands/local/cli_common/options.py @@ -92,6 +92,8 @@ def invoke_common_options(f): help="When specified, Lambda function container will start in debug mode and will expose this " "port on localhost.", envvar="SAM_DEBUG_PORT", + type=click.INT, + multiple=True, ), click.option( "--debugger-path", help="Host path to a debugger that will be mounted into the Lambda container." diff --git a/samcli/commands/local/invoke/cli.py b/samcli/commands/local/invoke/cli.py index 95b97f27b8..18b1a36d2b 100644 --- a/samcli/commands/local/invoke/cli.py +++ b/samcli/commands/local/invoke/cli.py @@ -136,7 +136,7 @@ def do_cli( # pylint: disable=R0914 docker_network=docker_network, log_file=log_file, skip_pull_image=skip_pull_image, - debug_port=debug_port, + debug_ports=debug_port, debug_args=debug_args, debugger_path=debugger_path, parameter_overrides=parameter_overrides, diff --git a/samcli/commands/local/lib/debug_context.py b/samcli/commands/local/lib/debug_context.py index 053b580255..a1b8fda3f6 100644 --- a/samcli/commands/local/lib/debug_context.py +++ b/samcli/commands/local/lib/debug_context.py @@ -4,14 +4,21 @@ class DebugContext: - def __init__(self, debug_port=None, debugger_path=None, debug_args=None): + def __init__(self, debug_ports=None, debugger_path=None, debug_args=None): + """ + Initialize the Debug Context with Lambda debugger options - self.debug_port = debug_port + :param tuple(int) debug_ports: Collection of debugger ports to be exposed from a docker container + :param Path debugger_path: Path to a debugger to be launched + :param string debug_args: Additional arguments to be passed to the debugger + """ + + self.debug_ports = debug_ports self.debugger_path = debugger_path self.debug_args = debug_args def __bool__(self): - return bool(self.debug_port) + return bool(self.debug_ports) def __nonzero__(self): return self.__bool__() diff --git a/samcli/commands/local/lib/local_lambda.py b/samcli/commands/local/lib/local_lambda.py index 199f253a95..c1ddbb4888 100644 --- a/samcli/commands/local/lib/local_lambda.py +++ b/samcli/commands/local/lib/local_lambda.py @@ -40,9 +40,10 @@ def __init__( :param samcli.commands.local.lib.provider.FunctionProvider function_provider: Provider that can return a Lambda function :param string cwd: Current working directory. We will resolve all function CodeURIs relative to this directory. - :param dict env_vars_values: Optional. Dictionary containing values of environment variables - :param integer debug_port: Optional. Port to bind the debugger to - :param string debug_args: Optional. Additional arguments passed to the debugger + :param string aws_profile: Optional. Name of the profile to fetch AWS credentials from. + :param string aws_region: Optional. AWS Region to use. + :param dict env_vars_values: Optional. Dictionary containing values of environment variables. + :param DebugContext debug_context: Optional. Debug context for the function (includes port, args, and path). """ self.local_runtime = local_runtime diff --git a/samcli/commands/local/start_api/cli.py b/samcli/commands/local/start_api/cli.py index 8063101359..83f699e33b 100644 --- a/samcli/commands/local/start_api/cli.py +++ b/samcli/commands/local/start_api/cli.py @@ -129,7 +129,7 @@ def do_cli( # pylint: disable=R0914 docker_network=docker_network, log_file=log_file, skip_pull_image=skip_pull_image, - debug_port=debug_port, + debug_ports=debug_port, debug_args=debug_args, debugger_path=debugger_path, parameter_overrides=parameter_overrides, diff --git a/samcli/commands/local/start_lambda/cli.py b/samcli/commands/local/start_lambda/cli.py index 0004df846b..b607febe2e 100644 --- a/samcli/commands/local/start_lambda/cli.py +++ b/samcli/commands/local/start_lambda/cli.py @@ -138,7 +138,7 @@ def do_cli( # pylint: disable=R0914 docker_network=docker_network, log_file=log_file, skip_pull_image=skip_pull_image, - debug_port=debug_port, + debug_ports=debug_port, debug_args=debug_args, debugger_path=debugger_path, parameter_overrides=parameter_overrides, diff --git a/samcli/local/docker/lambda_container.py b/samcli/local/docker/lambda_container.py index 8fa711d93d..2c17a3a195 100644 --- a/samcli/local/docker/lambda_container.py +++ b/samcli/local/docker/lambda_container.py @@ -92,20 +92,25 @@ def __init__( @staticmethod def _get_exposed_ports(debug_options): """ - Return Docker container port binding information. If a debug port is given, then we will ask Docker to - bind to same port both inside and outside the container ie. Runtime process is started in debug mode with + Return Docker container port binding information. If a debug port tuple is given, then we will ask Docker to + bind every given port to same port both inside and outside the container ie. Runtime process is started in debug mode with at given port inside the container and exposed to the host machine at the same port - :param int debug_port: Optional, integer value of debug port + :param DebugContext debug_options: Debugging options for the function (includes debug port, args, and path) :return dict: Dictionary containing port binding information. None, if debug_port was not given """ if not debug_options: return None - return { - # container port : host port - debug_options.debug_port: debug_options.debug_port - } + if not debug_options.debug_ports: + return None + + # container port : host port + ports_map = {} + for port in debug_options.debug_ports: + ports_map[port] = port + + return ports_map @staticmethod def _get_additional_options(runtime, debug_options): @@ -169,9 +174,8 @@ def _get_entry_point(runtime, debug_options=None): # pylint: disable=too-many-b Dockerfile. We override this default specifically when enabling debugging. The overridden entry point includes a few extra flags to start the runtime in debug mode. - :param string runtime: Lambda function runtime name - :param int debug_port: Optional, port for debugger - :param string debug_args: Optional additional arguments passed to the entry point. + :param string runtime: Lambda function runtime name. + :param DebugContext debug_options: Optional. Debug context for the function (includes port, args, and path). :return list: List containing the new entry points. Each element in the list is one portion of the command. ie. if command is ``node index.js arg1 arg2``, then this list will be ["node", "index.js", "arg1", "arg2"] """ @@ -179,7 +183,11 @@ def _get_entry_point(runtime, debug_options=None): # pylint: disable=too-many-b if not debug_options: return None - debug_port = debug_options.debug_port + debug_ports = debug_options.debug_ports + if not debug_ports: + return None + + debug_port = debug_ports[0] debug_args_list = [] if debug_options.debug_args: diff --git a/tests/functional/local/docker/test_lambda_container.py b/tests/functional/local/docker/test_lambda_container.py index 178487d142..b5e65f4e22 100644 --- a/tests/functional/local/docker/test_lambda_container.py +++ b/tests/functional/local/docker/test_lambda_container.py @@ -51,8 +51,8 @@ def setUp(self): self.expected_docker_image = self.IMAGE_NAME self.handler = "index.handler" self.layers = [] - self.debug_port = _rand_port() - self.debug_context = DebugContext(debug_port=self.debug_port, debugger_path=None, debug_args=None) + self.debug_port = [_rand_port()] + self.debug_context = DebugContext(debug_ports=self.debug_port, debugger_path=None, debug_args=None) self.code_dir = nodejs_lambda(self.HELLO_WORLD_CODE) self.network_prefix = "sam_cli_test_network" diff --git a/tests/unit/commands/local/cli_common/test_invoke_context.py b/tests/unit/commands/local/cli_common/test_invoke_context.py index ccc95d112b..2de5942c45 100644 --- a/tests/unit/commands/local/cli_common/test_invoke_context.py +++ b/tests/unit/commands/local/cli_common/test_invoke_context.py @@ -30,7 +30,7 @@ def test_must_read_from_necessary_files(self, SamFunctionProviderMock): docker_network="network", log_file=log_file, skip_pull_image=True, - debug_port=1111, + debug_ports=[1111], debugger_path="path-to-debugger", debug_args="args", parameter_overrides={}, @@ -73,7 +73,7 @@ def test_must_read_from_necessary_files(self, SamFunctionProviderMock): SamFunctionProviderMock.assert_called_with(template_dict, {"AWS::Region": "region"}) invoke_context._get_env_vars_value.assert_called_with(env_vars_file) invoke_context._setup_log_file.assert_called_with(log_file) - invoke_context._get_debug_context.assert_called_once_with(1111, "args", "path-to-debugger") + invoke_context._get_debug_context.assert_called_once_with([1111], "args", "path-to-debugger") invoke_context._get_container_manager.assert_called_once_with("network", True) @patch("samcli.commands.local.cli_common.invoke_context.SamFunctionProvider") @@ -171,7 +171,7 @@ def test_must_work_in_with_statement(self, ExitMock, EnterMock): docker_network="network", log_file="log_file", skip_pull_image=True, - debug_port=1111, + debug_ports=[1111], debugger_path="path-to-debugger", debug_args="args", aws_profile="profile", @@ -220,7 +220,7 @@ def setUp(self): log_file="log_file", skip_pull_image=True, force_image_build=True, - debug_port=1111, + debug_ports=[1111], debugger_path="path-to-debugger", debug_args="args", aws_profile="profile", @@ -287,7 +287,7 @@ def test_must_enable_auto_flush_if_debug( self, SamFunctionProviderMock, StreamWriterMock, osutils_stdout_mock, ExitMock ): - context = InvokeContext(template_file="template", debug_port=6000) + context = InvokeContext(template_file="template", debug_ports=[6000]) context._get_template_data = Mock() context._get_env_vars_value = Mock() @@ -391,7 +391,7 @@ def test_must_enable_auto_flush_if_debug( self, SamFunctionProviderMock, StreamWriterMock, osutils_stderr_mock, ExitMock ): - context = InvokeContext(template_file="template", debug_port=6000) + context = InvokeContext(template_file="template", debug_ports=[6000]) context._get_template_data = Mock() context._get_env_vars_value = Mock() @@ -571,7 +571,7 @@ def test_debugger_path_not_found(self, pathlib_mock): pathlib_mock.side_effect = error with self.assertRaises(DebugContextException): - InvokeContext._get_debug_context(debug_port=1111, debug_args=None, debugger_path="somepath") + InvokeContext._get_debug_context(debug_ports=[1111], debug_args=None, debugger_path="somepath") @patch("samcli.commands.local.cli_common.invoke_context.Path") def test_debugger_path_not_dir(self, pathlib_mock): @@ -582,13 +582,13 @@ def test_debugger_path_not_dir(self, pathlib_mock): pathlib_mock.return_value = pathlib_path_mock with self.assertRaises(DebugContextException): - InvokeContext._get_debug_context(debug_port=1111, debug_args=None, debugger_path="somepath") + InvokeContext._get_debug_context(debug_ports=1111, debug_args=None, debugger_path="somepath") def test_no_debug_port(self): debug_context = InvokeContext._get_debug_context(None, None, None) self.assertEqual(debug_context.debugger_path, None) - self.assertEqual(debug_context.debug_port, None) + self.assertEqual(debug_context.debug_ports, None) self.assertEqual(debug_context.debug_args, None) @patch("samcli.commands.local.cli_common.invoke_context.Path") @@ -596,17 +596,59 @@ def test_non_path_not_found_oserror_is_thrown(self, pathlib_mock): pathlib_mock.side_effect = OSError() with self.assertRaises(OSError): - InvokeContext._get_debug_context(debug_port=1111, debug_args=None, debugger_path="somepath") + InvokeContext._get_debug_context(debug_ports=1111, debug_args=None, debugger_path="somepath") @patch("samcli.commands.local.cli_common.invoke_context.DebugContext") def test_debug_port_given_without_debugger_path(self, debug_context_mock): debug_context_mock.return_value = "I am the DebugContext" - - debug_context = InvokeContext._get_debug_context(1111, None, None) + debug_context = InvokeContext._get_debug_context(debug_ports=1111, debug_args=None, debugger_path=None) self.assertEqual(debug_context, "I am the DebugContext") + debug_context_mock.assert_called_once_with(debug_ports=1111, debug_args=None, debugger_path=None) + + @patch("samcli.commands.local.cli_common.invoke_context.Path") + def test_debug_port_not_specified(self, pathlib_mock): + pathlib_path_mock = Mock() + pathlib_mock.return_value = pathlib_path_mock + + debug_context = InvokeContext._get_debug_context(debug_ports=None, debug_args=None, debugger_path="somepath") + self.assertEqual(None, debug_context.debug_ports) + + @patch("samcli.commands.local.cli_common.invoke_context.Path") + def test_debug_port_single_value_int(self, pathlib_mock): + pathlib_path_mock = Mock() + pathlib_mock.return_value = pathlib_path_mock + + debug_context = InvokeContext._get_debug_context(debug_ports=1111, debug_args=None, debugger_path="somepath") + self.assertEqual(1111, debug_context.debug_ports) - debug_context_mock.assert_called_once_with(debug_port=1111, debug_args=None, debugger_path=None) + @patch("samcli.commands.local.cli_common.invoke_context.Path") + def test_debug_port_single_value_string(self, pathlib_mock): + pathlib_path_mock = Mock() + pathlib_mock.return_value = pathlib_path_mock + + debug_context = InvokeContext._get_debug_context(debug_ports="1111", debug_args=None, debugger_path="somepath") + self.assertEqual("1111", debug_context.debug_ports) + + @patch("samcli.commands.local.cli_common.invoke_context.Path") + def test_debug_port_multiple_values_string(self, pathlib_mock): + pathlib_path_mock = Mock() + pathlib_mock.return_value = pathlib_path_mock + + debug_context = InvokeContext._get_debug_context( + debug_ports=["1111", "1112"], debug_args=None, debugger_path="somepath" + ) + self.assertEqual(["1111", "1112"], debug_context.debug_ports) + + @patch("samcli.commands.local.cli_common.invoke_context.Path") + def test_debug_port_multiple_values_int(self, pathlib_mock): + pathlib_path_mock = Mock() + pathlib_mock.return_value = pathlib_path_mock + + debug_context = InvokeContext._get_debug_context( + debug_ports=[1111, 1112], debug_args=None, debugger_path="somepath" + ) + self.assertEqual([1111, 1112], debug_context.debug_ports) @patch("samcli.commands.local.cli_common.invoke_context.DebugContext") @patch("samcli.commands.local.cli_common.invoke_context.Path") @@ -625,7 +667,7 @@ def test_debugger_path_resolves(self, pathlib_mock, debug_context_mock): self.assertEqual(debug_context, "I am the DebugContext") - debug_context_mock.assert_called_once_with(debug_port=1111, debug_args="args", debugger_path="full/path") + debug_context_mock.assert_called_once_with(debug_ports=1111, debug_args="args", debugger_path="full/path") resolve_path_mock.is_dir.assert_called_once() pathlib_path_mock.resolve.assert_called_once_with(strict=True) pathlib_mock.assert_called_once_with("./path") diff --git a/tests/unit/commands/local/invoke/test_cli.py b/tests/unit/commands/local/invoke/test_cli.py index d38e12b338..643f223294 100644 --- a/tests/unit/commands/local/invoke/test_cli.py +++ b/tests/unit/commands/local/invoke/test_cli.py @@ -25,7 +25,7 @@ def setUp(self): self.template = "template" self.eventfile = "eventfile" self.env_vars = "env-vars" - self.debug_port = 123 + self.debug_ports = [123] self.debug_args = "args" self.debugger_path = "/test/path" self.docker_volume_basedir = "basedir" @@ -60,7 +60,7 @@ def test_cli_must_setup_context_and_invoke(self, get_event_mock, InvokeContextMo event=self.eventfile, no_event=self.no_event, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, @@ -80,7 +80,7 @@ def test_cli_must_setup_context_and_invoke(self, get_event_mock, InvokeContextMo docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - debug_port=self.debug_port, + debug_ports=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, parameter_overrides=self.parameter_overrides, @@ -114,7 +114,7 @@ def test_cli_must_invoke_with_no_event(self, get_event_mock, InvokeContextMock): event=STDIN_FILE_NAME, no_event=self.no_event, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, @@ -134,7 +134,7 @@ def test_cli_must_invoke_with_no_event(self, get_event_mock, InvokeContextMock): docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - debug_port=self.debug_port, + debug_ports=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, parameter_overrides=self.parameter_overrides, @@ -167,7 +167,7 @@ def test_must_raise_user_exception_on_no_event_and_event(self, get_event_mock, I event=self.eventfile, no_event=self.no_event, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, @@ -215,7 +215,7 @@ def test_must_raise_user_exception_on_function_not_found( event=self.eventfile, no_event=self.no_event, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, @@ -263,7 +263,7 @@ def test_must_raise_user_exception_on_invalid_sam_template( event=self.eventfile, no_event=self.no_event, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, @@ -299,7 +299,7 @@ def test_must_raise_user_exception_on_invalid_env_vars(self, get_event_mock, Inv event=self.eventfile, no_event=self.no_event, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, diff --git a/tests/unit/commands/local/lib/test_debug_context.py b/tests/unit/commands/local/lib/test_debug_context.py index 9c19346e5a..9eb0a4d5de 100644 --- a/tests/unit/commands/local/lib/test_debug_context.py +++ b/tests/unit/commands/local/lib/test_debug_context.py @@ -9,20 +9,21 @@ class TestDebugContext(TestCase): def test_init(self): context = DebugContext("port", "debuggerpath", "debug_args") - self.assertEqual(context.debug_port, "port") + self.assertEqual(context.debug_ports, "port") self.assertEqual(context.debugger_path, "debuggerpath") self.assertEqual(context.debug_args, "debug_args") @parameterized.expand( [ ("1000", "debuggerpath", "debug_args"), - ("1000", None, None), - ("1000", None, "debug_args"), - ("1000", "debuggerpath", None), + (["1000"], "debuggerpath", "debug_args"), + (["1000", "1001"], "debuggerpath", "debug_args"), (1000, "debuggerpath", "debug_args"), - (1000, None, None), - (1000, None, "debug_args"), - (1000, "debuggerpath", None), + ([1000], "debuggerpath", "debug_args"), + ([1000, 1001], "debuggerpath", "debug_args"), + ([1000], None, None), + ([1000], None, "debug_args"), + ([1000], "debuggerpath", None), ] ) def test_bool_truthy(self, port, debug_path, debug_ars): @@ -46,13 +47,14 @@ def test_bool_falsy(self, port, debug_path, debug_ars): @parameterized.expand( [ ("1000", "debuggerpath", "debug_args"), - ("1000", None, None), - ("1000", None, "debug_args"), - ("1000", "debuggerpath", None), + (["1000"], "debuggerpath", "debug_args"), + (["1000", "1001"], "debuggerpath", "debug_args"), (1000, "debuggerpath", "debug_args"), - (1000, None, None), - (1000, None, "debug_args"), - (1000, "debuggerpath", None), + ([1000], "debuggerpath", "debug_args"), + ([1000, 1001], "debuggerpath", "debug_args"), + ([1000], None, None), + ([1000], None, "debug_args"), + ([1000], "debuggerpath", None), ] ) def test_nonzero_thruthy(self, port, debug_path, debug_ars): diff --git a/tests/unit/commands/local/start_api/test_cli.py b/tests/unit/commands/local/start_api/test_cli.py index e904944437..830cdbee1e 100644 --- a/tests/unit/commands/local/start_api/test_cli.py +++ b/tests/unit/commands/local/start_api/test_cli.py @@ -19,7 +19,7 @@ class TestCli(TestCase): def setUp(self): self.template = "template" self.env_vars = "env-vars" - self.debug_port = 123 + self.debug_ports = [123] self.debug_args = "args" self.debugger_path = "/test/path" self.docker_volume_basedir = "basedir" @@ -60,7 +60,7 @@ def test_cli_must_setup_context_and_start_service(self, local_api_service_mock, docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - debug_port=self.debug_port, + debug_ports=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, parameter_overrides=self.parameter_overrides, @@ -138,7 +138,7 @@ def call_cli(self): static_dir=self.static_dir, template=self.template, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, diff --git a/tests/unit/commands/local/start_lambda/test_cli.py b/tests/unit/commands/local/start_lambda/test_cli.py index 5dff100199..d34b5d1c09 100644 --- a/tests/unit/commands/local/start_lambda/test_cli.py +++ b/tests/unit/commands/local/start_lambda/test_cli.py @@ -15,7 +15,7 @@ class TestCli(TestCase): def setUp(self): self.template = "template" self.env_vars = "env-vars" - self.debug_port = 123 + self.debug_ports = [123] self.debug_args = "args" self.debugger_path = "/test/path" self.docker_volume_basedir = "basedir" @@ -55,7 +55,7 @@ def test_cli_must_setup_context_and_start_service(self, local_lambda_service_moc docker_network=self.docker_network, log_file=self.log_file, skip_pull_image=self.skip_pull_image, - debug_port=self.debug_port, + debug_ports=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, parameter_overrides=self.parameter_overrides, @@ -110,7 +110,7 @@ def call_cli(self): port=self.port, template=self.template, env_vars=self.env_vars, - debug_port=self.debug_port, + debug_port=self.debug_ports, debug_args=self.debug_args, debugger_path=self.debugger_path, docker_volume_basedir=self.docker_volume_basedir, diff --git a/tests/unit/local/docker/test_lambda_container.py b/tests/unit/local/docker/test_lambda_container.py index 30db16c83a..c20eb1de43 100644 --- a/tests/unit/local/docker/test_lambda_container.py +++ b/tests/unit/local/docker/test_lambda_container.py @@ -36,7 +36,7 @@ def setUp(self): self.code_dir = "codedir" self.env_var = {"var": "value"} self.memory_mb = 1024 - self.debug_options = DebugContext(debug_args="a=b c=d e=f", debug_port=1235) + self.debug_options = DebugContext(debug_args="a=b c=d e=f", debug_ports=[1235]) @patch.object(LambdaContainer, "_get_image") @patch.object(LambdaContainer, "_get_exposed_ports") @@ -108,12 +108,34 @@ def test_must_fail_for_unsupported_runtime(self): class TestLambdaContainer_get_exposed_ports(TestCase): def test_must_map_same_port_on_host_and_container(self): - debug_options = DebugContext(debug_port=12345) - expected = {debug_options.debug_port: debug_options.debug_port} + debug_options = DebugContext(debug_ports=[12345]) + expected = {port: port for port in debug_options.debug_ports} result = LambdaContainer._get_exposed_ports(debug_options) self.assertEqual(expected, result) + def test_must_map_multiple_ports_on_host_and_container(self): + + debug_options = DebugContext(debug_ports=[12345, 67890]) + expected = {port: port for port in debug_options.debug_ports} + result = LambdaContainer._get_exposed_ports(debug_options) + + self.assertEqual(expected, result) + + def test_empty_ports_list(self): + + debug_options = DebugContext(debug_ports=[]) + result = LambdaContainer._get_exposed_ports(debug_options) + + self.assertEqual(None, result) + + def test_none_ports_specified(self): + + debug_options = DebugContext(debug_ports=None) + result = LambdaContainer._get_exposed_ports(debug_options) + + self.assertEqual(None, result) + def test_must_skip_if_port_is_not_given(self): self.assertIsNone(LambdaContainer._get_exposed_ports(None), "No ports should be exposed") @@ -133,9 +155,9 @@ def test_must_return_lambci_image(self): class TestLambdaContainer_get_entry_point(TestCase): def setUp(self): - self.debug_port = 1235 + self.debug_ports = [1235] self.debug_args = "a=b c=d e=f" - self.debug_options = DebugContext(debug_port=1235, debug_args="a=b c=d e=f") + self.debug_options = DebugContext(debug_ports=[1235], debug_args="a=b c=d e=f") def test_must_skip_if_debug_port_is_not_specified(self): self.assertIsNone( @@ -176,7 +198,7 @@ def test_debug_arg_must_be_split_by_spaces_and_appended_to_bootstrap_based_entry @parameterized.expand([param(r) for r in RUNTIMES_WITH_ENTRYPOINT]) def test_must_provide_entrypoint_even_without_debug_args(self, runtime): - debug_options = DebugContext(debug_port=1235, debug_args=None) + debug_options = DebugContext(debug_ports=[1235], debug_args=None) result = LambdaContainer._get_entry_point(runtime, debug_options) self.assertIsNotNone(result) @@ -184,14 +206,14 @@ def test_must_provide_entrypoint_even_without_debug_args(self, runtime): class TestLambdaContainer_get_additional_options(TestCase): def test_no_additional_options_when_debug_options_is_none(self): - debug_options = DebugContext(debug_port=None) + debug_options = DebugContext(debug_ports=None) result = LambdaContainer._get_additional_options("runtime", debug_options) self.assertIsNone(result) @parameterized.expand([param(r) for r in RUNTIMES_WITH_ENTRYPOINT if not r.startswith("go")]) def test_default_value_returned_for_non_go_runtimes(self, runtime): - debug_options = DebugContext(debug_port=1235) + debug_options = DebugContext(debug_ports=[1235]) result = LambdaContainer._get_additional_options(runtime, debug_options) self.assertEqual(result, {}) @@ -200,7 +222,7 @@ def test_default_value_returned_for_non_go_runtimes(self, runtime): def test_go_runtime_returns_additional_options(self, runtime): expected = {"security_opt": ["seccomp:unconfined"], "cap_add": ["SYS_PTRACE"]} - debug_options = DebugContext(debug_port=1235) + debug_options = DebugContext(debug_ports=[1235]) result = LambdaContainer._get_additional_options(runtime, debug_options) self.assertEqual(result, expected) @@ -208,13 +230,13 @@ def test_go_runtime_returns_additional_options(self, runtime): class TestLambdaContainer_get_additional_volumes(TestCase): def test_no_additional_volumes_when_debug_options_is_none(self): - debug_options = DebugContext(debug_port=None) + debug_options = DebugContext(debug_ports=None) result = LambdaContainer._get_additional_volumes(debug_options) self.assertIsNone(result) def test_no_additional_volumes_when_debuggr_path_is_none(self): - debug_options = DebugContext(debug_port=1234) + debug_options = DebugContext(debug_ports=[1234]) result = LambdaContainer._get_additional_volumes(debug_options) self.assertIsNone(result) @@ -222,7 +244,7 @@ def test_no_additional_volumes_when_debuggr_path_is_none(self): def test_additional_volumes_returns_volume_with_debugger_path_is_set(self): expected = {"/somepath": {"bind": "/tmp/lambci_debug_files", "mode": "ro"}} - debug_options = DebugContext(debug_port=1234, debugger_path="/somepath") + debug_options = DebugContext(debug_ports=[1234], debugger_path="/somepath") result = LambdaContainer._get_additional_volumes(debug_options) self.assertEqual(result, expected) From 29300506828df5af3a3d3a2c90f4133ac07b6291 Mon Sep 17 00:00:00 2001 From: Trenton Lipscomb Date: Sun, 27 Oct 2019 14:06:35 +0900 Subject: [PATCH 28/49] Fixes empty account id in CloudWatch Scheduled Event --- .../events/cloudwatch/ScheduledEvent.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/local/lib/generated_sample_events/events/cloudwatch/ScheduledEvent.json b/samcli/commands/local/lib/generated_sample_events/events/cloudwatch/ScheduledEvent.json index b9783b607c..374ceaebea 100644 --- a/samcli/commands/local/lib/generated_sample_events/events/cloudwatch/ScheduledEvent.json +++ b/samcli/commands/local/lib/generated_sample_events/events/cloudwatch/ScheduledEvent.json @@ -2,7 +2,7 @@ "id": "cdc73f9d-aea9-11e3-9d5a-835b769c0d9c", "detail-type": "Scheduled Event", "source": "aws.events", - "account": "{{{account-id}}}", + "account": "{{{account_id}}}", "time": "1970-01-01T00:00:00Z", "region": "{{{region}}}", "resources": [ From 0c99a66c09b65084b996afc71ee29e9dce999e9f Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Fri, 31 May 2019 09:12:41 +0100 Subject: [PATCH 29/49] fix: Lambda timeout from parameter (#925) --- samcli/commands/local/lib/sam_function_provider.py | 12 ++++++++++-- .../local/invoke/test_integrations_cli.py | 6 +++++- tests/integration/testdata/invoke/template.yml | 13 ++++++++++++- .../local/lib/test_sam_function_provider.py | 8 ++++---- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/samcli/commands/local/lib/sam_function_provider.py b/samcli/commands/local/lib/sam_function_provider.py index 1e8365b2fa..59cd5a9665 100644 --- a/samcli/commands/local/lib/sam_function_provider.py +++ b/samcli/commands/local/lib/sam_function_provider.py @@ -2,9 +2,10 @@ Class that provides functions from a given SAM template """ +import ast import logging -from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn +from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn, InvalidSamTemplateException from .exceptions import InvalidLayerReference from .provider import FunctionProvider, Function, LayerVersion from .sam_base_provider import SamBaseProvider @@ -122,11 +123,18 @@ def _convert_sam_function_resource(name, resource_properties, layers): LOG.debug("Found Serverless function with name='%s' and CodeUri='%s'", name, codeuri) + timeout = resource_properties.get("Timeout") + if isinstance(timeout, str): + try: + timeout = ast.literal_eval(timeout) + except SyntaxError: + raise InvalidSamTemplateException('Invalid Number for Timeout: {}'.format(timeout)) + return Function( name=name, runtime=resource_properties.get("Runtime"), memory=resource_properties.get("MemorySize"), - timeout=resource_properties.get("Timeout"), + timeout=timeout, handler=resource_properties.get("Handler"), codeuri=codeuri, environment=resource_properties.get("Environment"), diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index d9999d5670..3af5b600fe 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -74,7 +74,11 @@ def test_invoke_of_lambda_function(self): @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") - @parameterized.expand([("TimeoutFunction"), ("TimeoutFunctionWithParameter")]) + @parameterized.expand([ + ("TimeoutFunction"), + ("TimeoutFunctionWithParameter"), + ("TimeoutFunctionWithStringParameter"), + ]) def test_invoke_with_timeout_set(self, function_name): command_list = self.get_command_list( function_name, template_path=self.template_path, event_path=self.event_path diff --git a/tests/integration/testdata/invoke/template.yml b/tests/integration/testdata/invoke/template.yml index fcb5493e74..f22754f67f 100644 --- a/tests/integration/testdata/invoke/template.yml +++ b/tests/integration/testdata/invoke/template.yml @@ -10,6 +10,10 @@ Parameters: MyRuntimeVersion: Type: String + StringValueTimeout: + Default: "5" + Type: Number + Resources: HelloWorldServerlessFunction: Type: AWS::Serverless::Function @@ -122,4 +126,11 @@ Resources: Timeout: !Ref DefaultTimeout MyRuntimeVersion: !Ref MyRuntimeVersion - + TimeoutFunctionWithStringParameter: + Type: AWS::Serverless::Function + Properties: + Handler: main.sleep_handler + Runtime: python3.6 + CodeUri: . + Timeout: + Ref: StringValueTimeout diff --git a/tests/unit/commands/local/lib/test_sam_function_provider.py b/tests/unit/commands/local/lib/test_sam_function_provider.py index 9b8e679159..65a031d8c6 100644 --- a/tests/unit/commands/local/lib/test_sam_function_provider.py +++ b/tests/unit/commands/local/lib/test_sam_function_provider.py @@ -244,7 +244,7 @@ def test_must_convert(self): "CodeUri": "/usr/local", "Runtime": "myruntime", "MemorySize": "mymemorysize", - "Timeout": "mytimeout", + "Timeout": "30", "Handler": "myhandler", "Environment": "myenvironment", "Role": "myrole", @@ -255,7 +255,7 @@ def test_must_convert(self): name="myname", runtime="myruntime", memory="mymemorysize", - timeout="mytimeout", + timeout=30, handler="myhandler", codeuri="/usr/local", environment="myenvironment", @@ -326,7 +326,7 @@ def test_must_convert(self): "Code": {"Bucket": "bucket"}, "Runtime": "myruntime", "MemorySize": "mymemorysize", - "Timeout": "mytimeout", + "Timeout": "30", "Handler": "myhandler", "Environment": "myenvironment", "Role": "myrole", @@ -337,7 +337,7 @@ def test_must_convert(self): name="myname", runtime="myruntime", memory="mymemorysize", - timeout="mytimeout", + timeout="30", handler="myhandler", codeuri=".", environment="myenvironment", From e5334daec194634b4f30bca8237f4a56e3ad548e Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Mon, 28 Oct 2019 11:50:50 -0500 Subject: [PATCH 30/49] update with HEAD of develop and add a unit test --- .../local/lib/sam_function_provider.py | 4 ++-- .../local/invoke/test_integrations_cli.py | 8 +++----- .../local/lib/test_sam_function_provider.py | 19 ++++++++++++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/samcli/commands/local/lib/sam_function_provider.py b/samcli/commands/local/lib/sam_function_provider.py index 59cd5a9665..6bc09c9fb7 100644 --- a/samcli/commands/local/lib/sam_function_provider.py +++ b/samcli/commands/local/lib/sam_function_provider.py @@ -127,8 +127,8 @@ def _convert_sam_function_resource(name, resource_properties, layers): if isinstance(timeout, str): try: timeout = ast.literal_eval(timeout) - except SyntaxError: - raise InvalidSamTemplateException('Invalid Number for Timeout: {}'.format(timeout)) + except ValueError: + raise InvalidSamTemplateException("Invalid Number for Timeout: {}".format(timeout)) return Function( name=name, diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index 3af5b600fe..9814c71b39 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -74,11 +74,9 @@ def test_invoke_of_lambda_function(self): @pytest.mark.flaky(reruns=3) @pytest.mark.timeout(timeout=300, method="thread") - @parameterized.expand([ - ("TimeoutFunction"), - ("TimeoutFunctionWithParameter"), - ("TimeoutFunctionWithStringParameter"), - ]) + @parameterized.expand( + [("TimeoutFunction"), ("TimeoutFunctionWithParameter"), ("TimeoutFunctionWithStringParameter")] + ) def test_invoke_with_timeout_set(self, function_name): command_list = self.get_command_list( function_name, template_path=self.template_path, event_path=self.event_path diff --git a/tests/unit/commands/local/lib/test_sam_function_provider.py b/tests/unit/commands/local/lib/test_sam_function_provider.py index 65a031d8c6..97c057dba8 100644 --- a/tests/unit/commands/local/lib/test_sam_function_provider.py +++ b/tests/unit/commands/local/lib/test_sam_function_provider.py @@ -2,7 +2,7 @@ from unittest.mock import patch from parameterized import parameterized -from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn +from samcli.commands.local.cli_common.user_exceptions import InvalidLayerVersionArn, InvalidSamTemplateException from samcli.commands.local.lib.provider import Function, LayerVersion from samcli.commands.local.lib.sam_function_provider import SamFunctionProvider from samcli.commands.local.lib.exceptions import InvalidLayerReference @@ -267,6 +267,23 @@ def test_must_convert(self): self.assertEqual(expected, result) + def test_must_fail_with_InvalidSamTemplateException(self): + + name = "myname" + properties = { + "CodeUri": "/usr/local", + "Runtime": "myruntime", + "MemorySize": "mymemorysize", + "Timeout": "timeout", + "Handler": "myhandler", + "Environment": "myenvironment", + "Role": "myrole", + "Layers": ["Layer1", "Layer2"], + } + + with self.assertRaises(InvalidSamTemplateException): + SamFunctionProvider._convert_sam_function_resource(name, properties, ["Layer1", "Layer2"]) + def test_must_skip_non_existent_properties(self): name = "myname" From c4025a3342bf115decd5c721a83c6c6ce1192d19 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 10:29:22 -0700 Subject: [PATCH 31/49] Improvements to fix linter issues in init --- samcli/commands/init/init_templates.py | 32 ++++++++++++------------- samcli/local/common/runtime_template.py | 12 ++++------ 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index f46de4cc6b..2fb5a07bc7 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -49,10 +49,9 @@ def prompt_for_location(self, runtime, dependency_manager): template_md = options[int(choice) - 1] # zero index if template_md.get("init_location") is not None: return template_md["init_location"] - elif template_md.get("directory") is not None: + if template_md.get("directory") is not None: return os.path.join(self.repo_path, template_md["directory"]) - else: - raise UserException("Invalid template. This should not be possible, please raise an issue.") + raise UserException("Invalid template. This should not be possible, please raise an issue.") def location_from_app_template(self, runtime, dependency_manager, app_template): options = self.init_options(runtime, dependency_manager) @@ -60,10 +59,9 @@ def location_from_app_template(self, runtime, dependency_manager, app_template): template = next(item for item in options if self._check_app_template(item, app_template)) if template.get("init_location") is not None: return template["init_location"] - elif template.get("directory") is not None: + if template.get("directory") is not None: return os.path.join(self.repo_path, template["directory"]) - else: - raise UserException("Invalid template. This should not be possible, please raise an issue.") + raise UserException("Invalid template. This should not be possible, please raise an issue.") except StopIteration: msg = "Can't find application template " + app_template + " - check valid values in interactive init." raise UserException(msg) @@ -126,10 +124,10 @@ def _git_executable(self): options = ["{}.cmd".format(execname), "{}.exe".format(execname), execname] else: options = [execname] - for name in options: - subprocess.Popen([name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # No exception. Let's pick this - return name + for name in options: + subprocess.Popen([name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # No exception. Let's pick this + return name def _should_clone_repo(self, expected_path): path = Path(expected_path) @@ -141,10 +139,10 @@ def _should_clone_repo(self, expected_path): return True self.repo_path = expected_path return False - else: - if self._no_interactive: - return self._auto_clone - do_clone = click.confirm( - "This process will clone app templates from https://github.com/awslabs/aws-sam-cli-app-templates - is this ok?" - ) - return do_clone + + if self._no_interactive: + return self._auto_clone + do_clone = click.confirm( + "This process will clone app templates from https://github.com/awslabs/aws-sam-cli-app-templates - is this ok?" + ) + return do_clone diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index 8db49faa08..eda4ae3802 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -89,13 +89,11 @@ "java8": ["maven", "gradle"], } -SUPPORTED_DEP_MANAGERS = set( - [ - c["dependency_manager"] - for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))) - if c["dependency_manager"] - ] -) +SUPPORTED_DEP_MANAGERS = { + c["dependency_manager"] + for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))) + if c["dependency_manager"] +} RUNTIMES = set( itertools.chain(*[c["runtimes"] for c in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values())))]) From d34ea10c9a51b84831fd97bcd1e6e06225e4bbed Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 10:57:39 -0700 Subject: [PATCH 32/49] Update Mock Import --- tests/unit/commands/init/test_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index ceb4ce376c..77e494d84c 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -2,7 +2,7 @@ import subprocess import click -from mock import mock_open, patch, PropertyMock, MagicMock +from unittest.mock import mock_open, patch, PropertyMock, MagicMock from re import search from unittest import TestCase From 7061eb6554c908c1f760f3c6efc04ae7ff8bd554 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 12:24:23 -0700 Subject: [PATCH 33/49] Try batch file for git --- samcli/commands/init/init_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 2fb5a07bc7..9d879286f9 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -121,7 +121,7 @@ def _clone_repo(self): def _git_executable(self): execname = "git" if platform.system().lower() == "windows": - options = ["{}.cmd".format(execname), "{}.exe".format(execname), execname] + options = ["{}.cmd".format(execname), "{}.exe".format(execname), "{}.bat".format(execname), execname] else: options = [execname] for name in options: From 7cf2e35b1fb46f4cc2d9b8544206b5c8f70aa3ad Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 12:55:27 -0700 Subject: [PATCH 34/49] Make Unit Test for Init Windows-Compatible Slashes behave differently. --- tests/unit/commands/init/test_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index 77e494d84c..7988de72d4 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -32,7 +32,7 @@ def test_location_from_app_template(self, subprocess_mock, git_exec_mock): mock_cfg.return_value = "/tmp/test-sam" with patch("samcli.commands.init.init_templates.open", m): location = it.location_from_app_template("ruby2.5", "bundler", "hello-world") - self.assertEqual(location, "/tmp/test-sam/aws-sam-cli-app-templates/mock-ruby-template") + self.assertTrue(search("mock-ruby-template", location)) @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") @patch("click.prompt") From cc3cba0240ed56f2edd96dbc4df9f949580e87a7 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 14:24:50 -0700 Subject: [PATCH 35/49] Switch execname order Checking integration test behavior. --- samcli/commands/init/init_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 9d879286f9..c4facde2dd 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -121,7 +121,7 @@ def _clone_repo(self): def _git_executable(self): execname = "git" if platform.system().lower() == "windows": - options = ["{}.cmd".format(execname), "{}.exe".format(execname), "{}.bat".format(execname), execname] + options = [execname, "{}.cmd".format(execname), "{}.exe".format(execname), "{}.bat".format(execname)] else: options = [execname] for name in options: From 5053efadaf1450d4300f08bc9395cabea3646eab Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 14:34:03 -0700 Subject: [PATCH 36/49] Catch exceptions when checking git executables Had a bug where the wrong executable name would lead to a premature exit. This change fixes that. --- samcli/commands/init/init_templates.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index c4facde2dd..93129618c8 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -5,6 +5,7 @@ import itertools import json import os +import logging import platform import shutil import subprocess @@ -17,6 +18,8 @@ from samcli.commands.exceptions import UserException from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING +LOG = logging.getLogger(__name__) + class InitTemplates: def __init__(self, no_interactive=False, auto_clone=True): @@ -125,9 +128,12 @@ def _git_executable(self): else: options = [execname] for name in options: - subprocess.Popen([name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # No exception. Let's pick this - return name + try: + subprocess.Popen([name], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # No exception. Let's pick this + return name + except OSError as ex: + LOG.debug("Unable to find executable %s", name, exc_info=ex) def _should_clone_repo(self, expected_path): path = Path(expected_path) From 0ec8bb217ad92220fc9b16c90b57f115bb6b3eb0 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 14:42:04 -0700 Subject: [PATCH 37/49] Add explicit exception for missing git Will better align with errors in place. --- samcli/commands/init/init_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 93129618c8..2d985249d5 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -134,6 +134,7 @@ def _git_executable(self): return name except OSError as ex: LOG.debug("Unable to find executable %s", name, exc_info=ex) + raise OSError("Cannot find git, was looking at executables: {}".format(options)) def _should_clone_repo(self, expected_path): path = Path(expected_path) From 7624b2043de741a985202a1d7328e65c15aec73c Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 15:13:37 -0700 Subject: [PATCH 38/49] Add git executable unit test --- tests/unit/commands/init/test_templates.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index 7988de72d4..08552dcfe0 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -57,3 +57,11 @@ def test_fallback_process_error(self, git_exec_mock, prompt_mock): it = InitTemplates(True) location = it.prompt_for_location("ruby2.5", "bundler") self.assertTrue(search("cookiecutter-aws-sam-hello-ruby", location)) + + def test_git_executable_windows(self): + with patch("platform.system", new_callable=MagicMock) as mock_platform: + mock_platform.return_value = "Windows" + with patch("subprocess.Popen", new_callable=MagicMock) as mock_popen: + it = InitTemplates(True) + executable = it._git_executable() + self.assertEqual(executable, "git") From efcc51f90b6397d23fda58948a31b6dca6fdbda9 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 15:50:27 -0700 Subject: [PATCH 39/49] Add git executable check --- appveyor-windows.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/appveyor-windows.yml b/appveyor-windows.yml index 0bddf396cf..4ece10a550 100644 --- a/appveyor-windows.yml +++ b/appveyor-windows.yml @@ -60,6 +60,9 @@ install: # Switch to Docker Linux containers - ps: Switch-DockerLinux + # Check for git executable + - "git --version" + # Echo final Path - "echo %PATH%" From 4b763854a3f6b8dd34e0d58fc75b4cb72d877428 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 16:45:16 -0700 Subject: [PATCH 40/49] Add Exception Info when Git Fails This is a WIP to fix breaking Windows integ tests. --- samcli/commands/init/init_templates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 2d985249d5..9deaeef4f1 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -113,8 +113,8 @@ def _clone_repo(self): [self._git_executable(), "clone", self._repo_url], cwd=shared_dir, stderr=subprocess.STDOUT ) self.repo_path = expected_path - except OSError: - click.echo("WARN: Can't clone app repo, git executable not found.") + except OSError as ex: + LOG.warn("WARN: Can't clone app repo, git executable not found", exc_info=ex) except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): From b3594a088ecde4fd63565110418ffb267ce532e5 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 16:52:11 -0700 Subject: [PATCH 41/49] Fix Warning Method This should have been part of the last commit. --- samcli/commands/init/init_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 9deaeef4f1..a851b4beac 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -114,7 +114,7 @@ def _clone_repo(self): ) self.repo_path = expected_path except OSError as ex: - LOG.warn("WARN: Can't clone app repo, git executable not found", exc_info=ex) + LOG.warning("WARN: Can't clone app repo, git executable not found", exc_info=ex) except subprocess.CalledProcessError as clone_error: output = clone_error.output.decode("utf-8") if "not found" in output.lower(): From 3382135c40a558e5403035457c22e84d0056b119 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 17:15:48 -0700 Subject: [PATCH 42/49] Run git in Appveyor test stage --- appveyor-windows.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor-windows.yml b/appveyor-windows.yml index 4ece10a550..e4a64727ec 100644 --- a/appveyor-windows.yml +++ b/appveyor-windows.yml @@ -68,6 +68,7 @@ install: test_script: # Reactivate virtualenv before running tests + - "git --version" - "venv\\Scripts\\activate" - "docker system prune -a -f" - "pytest -vv tests/integration" From f8d7977c0837fe8f1cba6d2a538b17f4d32b9f96 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Mon, 28 Oct 2019 17:24:31 -0700 Subject: [PATCH 43/49] WIP: Create Shared Directory If the shared directory doesn't exist, create it before cloning the template git repo. This will need to be refactored as some layer code does the same thing. --- samcli/commands/init/init_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index a851b4beac..72e32c3dee 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -106,6 +106,7 @@ def _init_options_from_bundle(self, runtime, dependency_manager): def _clone_repo(self): shared_dir = global_cfg.config_dir + shared_dir.mkdir(mode=0o700, parents=True, exist_ok=True) expected_path = os.path.normpath(os.path.join(shared_dir, self._repo_name)) if self._should_clone_repo(expected_path): try: From 7255c8822110d988a36ea170f6cefc4e4f97223f Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 29 Oct 2019 09:19:54 -0700 Subject: [PATCH 44/49] Create Shared Dir if Necessary Init code assumed existence of shared directory, which is likely but not guaranteed. --- samcli/commands/init/init_templates.py | 11 ++++++++++- tests/unit/commands/init/test_cli.py | 21 ++++++++++++++------- tests/unit/commands/init/test_templates.py | 9 ++++++--- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 72e32c3dee..f6a5e9cb79 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -104,9 +104,18 @@ def _init_options_from_bundle(self, runtime, dependency_manager): ) raise UserException(msg) + def _shared_dir_check(self, shared_dir): + try: + shared_dir.mkdir(mode=0x700, parents=True, exist_ok=True) + return True + except OSError as ex: + LOG.warning("WARN: Unable to create shared directory.", exc_info=ex) + return False + def _clone_repo(self): shared_dir = global_cfg.config_dir - shared_dir.mkdir(mode=0o700, parents=True, exist_ok=True) + if not self._shared_dir_check(shared_dir): + return expected_path = os.path.normpath(os.path.join(shared_dir, self._repo_name)) if self._should_clone_repo(expected_path): try: diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index 7d60389d45..98dc5e1b88 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -23,8 +23,9 @@ def setUp(self): self.no_input = False self.extra_context = {"project_name": "testing project", "runtime": "python3.6"} + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @patch("samcli.commands.init.init_generator.generate_project") - def test_init_cli(self, generate_project_patch): + def test_init_cli(self, generate_project_patch, sd_mock): # GIVEN generate_project successfully created a project # WHEN a project name has been passed init_cli( @@ -52,7 +53,8 @@ def test_init_cli(self, generate_project_patch): self.extra_context, ) - def test_init_fails_invalid_template(self): + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") + def test_init_fails_invalid_template(self, sd_mock): # WHEN an unknown app template is passed in # THEN an exception should be raised with self.assertRaises(UserException): @@ -69,7 +71,8 @@ def test_init_fails_invalid_template(self): auto_clone=False, ) - def test_init_fails_invalid_dep_mgr(self): + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") + def test_init_fails_invalid_dep_mgr(self, sd_mock): # WHEN an unknown app template is passed in # THEN an exception should be raised with self.assertRaises(UserException): @@ -86,8 +89,9 @@ def test_init_fails_invalid_dep_mgr(self): auto_clone=False, ) + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @patch("samcli.commands.init.init_generator.generate_project") - def test_init_cli_interactive(self, generate_project_patch): + def test_init_cli_interactive(self, generate_project_patch, sd_mock): # WHEN the user follows interactive init prompts # 1: selecting managed templates @@ -121,8 +125,9 @@ def test_init_cli_interactive(self, generate_project_patch): {"project_name": "test-project", "runtime": "ruby2.5"}, ) + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @patch("samcli.commands.init.init_generator.generate_project") - def test_init_cli_int_with_app_template(self, generate_project_patch): + def test_init_cli_int_with_app_template(self, generate_project_patch, sd_mock): # WHEN the user follows interactive init prompts # test-project: response to name @@ -153,8 +158,9 @@ def test_init_cli_int_with_app_template(self, generate_project_patch): {"project_name": "test-project", "runtime": "ruby2.5"}, ) + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @patch("samcli.commands.init.init_generator.generate_project") - def test_init_cli_int_from_location(self, generate_project_patch): + def test_init_cli_int_from_location(self, generate_project_patch, sd_mock): # WHEN the user follows interactive init prompts # 2: selecting custom location @@ -216,8 +222,9 @@ def test_init_cli_mutually_exclusive_params_fails(self): auto_clone=False, ) + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @patch("samcli.commands.init.init_generator.generate_project") - def test_init_cli_generate_project_fails(self, generate_project_patch): + def test_init_cli_generate_project_fails(self, generate_project_patch, sd_mock): # GIVEN generate_project fails to create a project generate_project_patch.side_effect = GenerateProjectFailedError( diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index 08552dcfe0..b0ae8e7202 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -12,7 +12,8 @@ class TestTemplates(TestCase): @patch("subprocess.check_output") @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") - def test_location_from_app_template(self, subprocess_mock, git_exec_mock): + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") + def test_location_from_app_template(self, subprocess_mock, git_exec_mock, sd_mock): it = InitTemplates(True) manifest = { @@ -36,7 +37,8 @@ def test_location_from_app_template(self, subprocess_mock, git_exec_mock): @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") @patch("click.prompt") - def test_fallback_options(self, git_exec_mock, prompt_mock): + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") + def test_fallback_options(self, git_exec_mock, prompt_mock, sd_mock): prompt_mock.return_value = "1" with patch("subprocess.check_output", new_callable=MagicMock) as mock_sub: with patch("samcli.cli.global_config.GlobalConfig.config_dir", new_callable=PropertyMock) as mock_cfg: @@ -48,7 +50,8 @@ def test_fallback_options(self, git_exec_mock, prompt_mock): @patch("samcli.commands.init.init_templates.InitTemplates._git_executable") @patch("click.prompt") - def test_fallback_process_error(self, git_exec_mock, prompt_mock): + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") + def test_fallback_process_error(self, git_exec_mock, prompt_mock, sd_mock): prompt_mock.return_value = "1" with patch("subprocess.check_output", new_callable=MagicMock) as mock_sub: with patch("samcli.cli.global_config.GlobalConfig.config_dir", new_callable=PropertyMock) as mock_cfg: From 957e80206fb6bf7074953c733a459d504aa9a55e Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 29 Oct 2019 09:47:23 -0700 Subject: [PATCH 45/49] Add Shared Dir Unit Tests --- tests/unit/commands/init/test_templates.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index b0ae8e7202..348e653920 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -68,3 +68,14 @@ def test_git_executable_windows(self): it = InitTemplates(True) executable = it._git_executable() self.assertEqual(executable, "git") + + def test_shared_dir_check(self): + it = InitTemplates(True) + shared_dir_mock = MagicMock() + self.assertTrue(it._shared_dir_check(shared_dir_mock)) + + def test_shared_dir_failure(self): + it = InitTemplates(True) + shared_dir_mock = MagicMock() + shared_dir_mock.mkdir.side_effect = OSError("fail") + self.assertFalse(it._shared_dir_check(shared_dir_mock)) From 216b8027e440e6ca815a1eecc42eafb85286603a Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 29 Oct 2019 10:29:58 -0700 Subject: [PATCH 46/49] Add One More Test --- tests/unit/commands/init/test_templates.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index 348e653920..6f10ca61e1 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -69,6 +69,13 @@ def test_git_executable_windows(self): executable = it._git_executable() self.assertEqual(executable, "git") + def test_git_executable_fails(self): + with patch("subprocess.Popen", new_callable=MagicMock) as mock_popen: + mock_popen.side_effect = OSError("fail") + it = InitTemplates(True) + with self.assertRaises(OSError): + executable = it._git_executable() + def test_shared_dir_check(self): it = InitTemplates(True) shared_dir_mock = MagicMock() From 8fe22cc02da6402c0190bff36e8b799e22af677d Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 29 Oct 2019 11:54:38 -0700 Subject: [PATCH 47/49] Move Local Imports into do_cli --- samcli/commands/init/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 340e71e0ea..da7ff13e14 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -2,21 +2,13 @@ """ Init command to scaffold a project app from a template """ -import itertools -import json import logging -import os -import subprocess import click from samcli.cli.main import pass_context, common_options, global_cfg -from samcli.commands.exceptions import UserException from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS from samcli.lib.telemetry.metrics import track_command -from samcli.commands.init.init_generator import do_generate -from samcli.commands.init.init_templates import InitTemplates -from samcli.commands.init.interactive_init_flow import do_interactive LOG = logging.getLogger(__name__) @@ -61,6 +53,7 @@ def cli(ctx, no_interactive, location, runtime, dependency_manager, output_dir, ) # pragma: no cover +# pylint: disable=too-many-locals def do_cli( ctx, no_interactive, @@ -73,6 +66,11 @@ def do_cli( no_input, auto_clone=True, ): + from samcli.commands.exceptions import UserException + from samcli.commands.init.init_generator import do_generate + from samcli.commands.init.init_templates import InitTemplates + from samcli.commands.init.interactive_init_flow import do_interactive + # check for mutually exclusive parameters if location and app_template: msg = """ From 402d8bf0b0cb3bc7604821a9015ddf30c3ffd0cb Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Tue, 29 Oct 2019 14:07:50 -0700 Subject: [PATCH 48/49] chore: bump version to 0.30.0 --- samcli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/__init__.py b/samcli/__init__.py index 6ae52e67dd..d126c85d2a 100644 --- a/samcli/__init__.py +++ b/samcli/__init__.py @@ -2,4 +2,4 @@ SAM CLI version """ -__version__ = "0.23.0" +__version__ = "0.30.0" From e30ce9fe7017f0ff8d4855221903be6b4037abd3 Mon Sep 17 00:00:00 2001 From: Alex Wood Date: Wed, 30 Oct 2019 12:46:51 -0700 Subject: [PATCH 49/49] Add HELP_TEXT for init command (#1492) * Add HELP_TEXT for init command * Reformat with Black --- samcli/commands/init/__init__.py | 46 ++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index da7ff13e14..79a0966a5f 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -12,9 +12,51 @@ LOG = logging.getLogger(__name__) +HELP_TEXT = """ \b + Initialize a serverless application with a SAM template, folder + structure for your Lambda functions, connected to an event source such as APIs, + S3 Buckets or DynamoDB Tables. This application includes everything you need to + get started with serverless and eventually grow into a production scale application. + \b + This command can initialize a boilerplate serverless app. If you want to create your own + template as well as use a custom location please take a look at our official documentation. +\b +Common usage: + \b + Starts an interactive prompt process to initialize a new project: + \b + $ sam init + \b + Initializes a new SAM project using project templates without an interactive workflow: + \b + $ sam init --name sam-app --runtime nodejs10.x --dependency-manager npm --app-template hello-world + \b + Initializes a new SAM project using custom template in a Git/Mercurial repository + \b + # gh being expanded to github url + $ sam init --location gh:aws-samples/cookiecutter-aws-sam-python + \b + $ sam init --location git+ssh://git@github.com/aws-samples/cookiecutter-aws-sam-python.git + \b + $ sam init --location hg+ssh://hg@bitbucket.org/repo/template-name + \b + Initializes a new SAM project using custom template in a Zipfile + \b + $ sam init --location /path/to/template.zip + \b + $ sam init --location https://example.com/path/to/template.zip + \b + Initializes a new SAM project using custom template in a local path + \b + $ sam init --location /path/to/template/folder +""" + @click.command( - "init", short_help="Init an AWS SAM application.", context_settings=dict(help_option_names=["-h", "--help"]) + "init", + help=HELP_TEXT, + short_help="Init an AWS SAM application.", + context_settings=dict(help_option_names=["-h", "--help"]), ) @click.option( "--no-interactive", @@ -77,7 +119,7 @@ def do_cli( You must not provide both the --location and --app-template parameters. You can run 'sam init' without any options for an interactive initialization flow, or you can provide one of the following required parameter combinations: - --name and --runtime and --app-template + --name and --runtime and --app-template and --dependency-manager --location """ raise UserException(msg)