From 9e723d286fa201eb09152ce83a36b999d3eada27 Mon Sep 17 00:00:00 2001 From: Anton Khodak Date: Mon, 5 Feb 2018 20:27:31 +0200 Subject: [PATCH 1/4] Add default timeout for different Python versions --- cwltest/__init__.py | 83 ++++++++++++++++++++++++--------------------- cwltest/utils.py | 29 +++++++++++++++- requirements.txt | 1 + setup.py | 2 ++ 4 files changed, 76 insertions(+), 39 deletions(-) diff --git a/cwltest/__init__.py b/cwltest/__init__.py index d3a35a5..e861544 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -3,66 +3,52 @@ from __future__ import absolute_import from __future__ import print_function -from six.moves import range -from six.moves import zip - import argparse import json +import logging import os +import pipes +import shutil import subprocess import sys -import shutil import tempfile +import threading +import time + import junit_xml import ruamel.yaml as yaml import ruamel.yaml.scanner as yamlscanner -import pipes -import logging import schema_salad.ref_resolver -import time -import threading from concurrent.futures import ThreadPoolExecutor -from typing import Any, Dict, List, Text +from six.moves import range +from six.moves import zip +from typing import Any, Dict, List, Callable -from cwltest.utils import compare, CompareFail +from cwltest.utils import compare, CompareFail, TestResult _logger = logging.getLogger("cwltest") _logger.addHandler(logging.StreamHandler()) _logger.setLevel(logging.INFO) UNSUPPORTED_FEATURE = 33 -RUNTIME = sys.version_info.major - - -class TestResult(object): +DEFAULT_TIMEOUT = 900 # 15 minutes - """Encapsulate relevant test result data.""" - - def __init__(self, return_code, standard_output, error_output, duration, classname, message=''): - # type: (int, Text, Text, float, Text, str) -> None - self.return_code = return_code - self.standard_output = standard_output - self.error_output = error_output - self.duration = duration - self.message = message - self.classname = classname - - def create_test_case(self, test): - # type: (Dict[Text, Any]) -> junit_xml.TestCase - doc = test.get(u'doc', 'N/A').strip() - case = junit_xml.TestCase( - doc, elapsed_sec=self.duration, classname=self.classname, - stdout=self.standard_output, stderr=self.error_output, - ) - if self.return_code > 0: - case.failure_message = self.message - return case +if sys.version_info >= (3, 3): + # use a much simpler solution with Python >= 3.3 + timeout_exc = subprocess.TimeoutExpired +else: + import timeout_decorator + class TimeoutException(Exception): + pass + timeout_exc = TimeoutException templock = threading.Lock() -def run_test(args, i, tests): # type: (argparse.Namespace, int, List[Dict[str, str]]) -> TestResult +def run_test(args, i, tests, run_test_command, timeout): + # type: (argparse.Namespace, int, List[Dict[str, str]], Callable, int) -> TestResult + global templock out = {} # type: Dict[str,Any] @@ -105,7 +91,7 @@ def run_test(args, i, tests): # type: (argparse.Namespace, int, List[Dict[str, start_time = time.time() stderr = subprocess.PIPE if not args.verbose else None process = subprocess.Popen(test_command, stdout=subprocess.PIPE, stderr=stderr) - outstr, outerr = [var.decode('utf-8') for var in process.communicate()] + outstr, outerr = run_test_command(process) return_code = process.poll() duration = time.time() - start_time if return_code: @@ -135,6 +121,10 @@ def run_test(args, i, tests): # type: (argparse.Namespace, int, List[Dict[str, except KeyboardInterrupt: _logger.error(u"""Test interrupted: %s""", " ".join([pipes.quote(tc) for tc in test_command])) raise + except timeout_exc: + _logger.error(u"""Test timed out: %s""", " ".join([pipes.quote(tc) for tc in test_command])) + _logger.error(t.get("doc")) + return TestResult(2, outstr, outerr, timeout, args.classname, "Test timed out") fail_message = '' @@ -177,6 +167,9 @@ def main(): # type: () -> int "(defaults to one).") parser.add_argument("--verbose", action="store_true", help="More verbose output during test run.") parser.add_argument("--classname", type=str, default="", help="Specify classname for the Test Suite.") + parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="Time of execution in seconds after " + "which the test will be skipped." + "Defaults to 900 sec (15 minutes)") args = parser.parse_args() if '--' in args.args: @@ -227,9 +220,23 @@ def main(): # type: () -> int else: ntest = list(range(0, len(tests))) + if sys.version_info >= (3, 3): + + def run_test_command_py3(process): + return [var.decode('utf-8') for var in process.communicate(timeout=args.timeout)] + + run_test_command = run_test_command_py3 + else: + + @timeout_decorator.timeout(args.timeout, use_signals=False, timeout_exception=timeout_exc) + def run_test_command_py2(process): + return [var.decode('utf-8') for var in process.communicate()] + + run_test_command = run_test_command_py2 + total = 0 with ThreadPoolExecutor(max_workers=args.j) as executor: - jobs = [executor.submit(run_test, args, i, tests) + jobs = [executor.submit(run_test, args, i, tests, run_test_command, args.timeout) for i in ntest] try: for i, job in zip(ntest, jobs): diff --git a/cwltest/utils.py b/cwltest/utils.py index 77fb7be..19f8033 100644 --- a/cwltest/utils.py +++ b/cwltest/utils.py @@ -1,9 +1,36 @@ import json -from typing import Any, Dict, Set + +import junit_xml +from typing import Any, Dict, Set, Text from six.moves import range +class TestResult(object): + + """Encapsulate relevant test result data.""" + + def __init__(self, return_code, standard_output, error_output, duration, classname, message=''): + # type: (int, Text, Text, float, Text, str) -> None + self.return_code = return_code + self.standard_output = standard_output + self.error_output = error_output + self.duration = duration + self.message = message + self.classname = classname + + def create_test_case(self, test): + # type: (Dict[Text, Any]) -> junit_xml.TestCase + doc = test.get(u'doc', 'N/A').strip() + case = junit_xml.TestCase( + doc, elapsed_sec=self.duration, classname=self.classname, + stdout=self.standard_output, stderr=self.error_output, + ) + if self.return_code > 0: + case.failure_message = self.message + return case + + class CompareFail(Exception): @classmethod diff --git a/requirements.txt b/requirements.txt index 7ca9fd3..db288ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ schema-salad >= 1.14 typing>=3.6,<3.7; python_version < '3.5' futures >= 3.0.5; python_version == '2.7' junit-xml >= 1.7 +timeout_decorator>=0.4; python_version < '3.3' diff --git a/setup.py b/setup.py index 81cab1b..1da9437 100755 --- a/setup.py +++ b/setup.py @@ -28,6 +28,8 @@ if sys.version_info[:2] < (3, 5): install_requires.append('typing >= 3.5.2') +if sys.version_info[:2] < (3, 3): + install_requires.append('timeout_decorator >= 0.4.0') setup(name='cwltest', version='1.0', From cbefb212246c56a8d054ae4c104bd3ce8f616026 Mon Sep 17 00:00:00 2001 From: Anton Khodak Date: Mon, 5 Feb 2018 21:04:08 +0200 Subject: [PATCH 2/4] Use backport of subprocess for Python2 --- cwltest/__init__.py | 38 +++++++++----------------------------- requirements.txt | 3 ++- setup.py | 5 +---- 3 files changed, 12 insertions(+), 34 deletions(-) diff --git a/cwltest/__init__.py b/cwltest/__init__.py index e861544..a7a498c 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -9,7 +9,6 @@ import os import pipes import shutil -import subprocess import sys import tempfile import threading @@ -22,7 +21,7 @@ from concurrent.futures import ThreadPoolExecutor from six.moves import range from six.moves import zip -from typing import Any, Dict, List, Callable +from typing import Any, Dict, List from cwltest.utils import compare, CompareFail, TestResult @@ -33,21 +32,16 @@ UNSUPPORTED_FEATURE = 33 DEFAULT_TIMEOUT = 900 # 15 minutes -if sys.version_info >= (3, 3): - # use a much simpler solution with Python >= 3.3 - timeout_exc = subprocess.TimeoutExpired +if sys.version_info.major < 3: + import subprocess32 as subprocess else: - import timeout_decorator - - class TimeoutException(Exception): - pass - timeout_exc = TimeoutException + import subprocess templock = threading.Lock() -def run_test(args, i, tests, run_test_command, timeout): - # type: (argparse.Namespace, int, List[Dict[str, str]], Callable, int) -> TestResult +def run_test(args, i, tests, timeout): + # type: (argparse.Namespace, int, List[Dict[str, str]], int) -> TestResult global templock @@ -91,7 +85,7 @@ def run_test(args, i, tests, run_test_command, timeout): start_time = time.time() stderr = subprocess.PIPE if not args.verbose else None process = subprocess.Popen(test_command, stdout=subprocess.PIPE, stderr=stderr) - outstr, outerr = run_test_command(process) + outstr, outerr = [var.decode('utf-8') for var in process.communicate(timeout=timeout)] return_code = process.poll() duration = time.time() - start_time if return_code: @@ -121,7 +115,7 @@ def run_test(args, i, tests, run_test_command, timeout): except KeyboardInterrupt: _logger.error(u"""Test interrupted: %s""", " ".join([pipes.quote(tc) for tc in test_command])) raise - except timeout_exc: + except subprocess.TimeoutExpired: _logger.error(u"""Test timed out: %s""", " ".join([pipes.quote(tc) for tc in test_command])) _logger.error(t.get("doc")) return TestResult(2, outstr, outerr, timeout, args.classname, "Test timed out") @@ -220,23 +214,9 @@ def main(): # type: () -> int else: ntest = list(range(0, len(tests))) - if sys.version_info >= (3, 3): - - def run_test_command_py3(process): - return [var.decode('utf-8') for var in process.communicate(timeout=args.timeout)] - - run_test_command = run_test_command_py3 - else: - - @timeout_decorator.timeout(args.timeout, use_signals=False, timeout_exception=timeout_exc) - def run_test_command_py2(process): - return [var.decode('utf-8') for var in process.communicate()] - - run_test_command = run_test_command_py2 - total = 0 with ThreadPoolExecutor(max_workers=args.j) as executor: - jobs = [executor.submit(run_test, args, i, tests, run_test_command, args.timeout) + jobs = [executor.submit(run_test, args, i, tests, args.timeout) for i in ntest] try: for i, job in zip(ntest, jobs): diff --git a/requirements.txt b/requirements.txt index db288ec..600a7db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ schema-salad >= 1.14 typing>=3.6,<3.7; python_version < '3.5' futures >= 3.0.5; python_version == '2.7' +subprocess32; python_version < '3' junit-xml >= 1.7 -timeout_decorator>=0.4; python_version < '3.3' + diff --git a/setup.py b/setup.py index 1da9437..2b16faf 100755 --- a/setup.py +++ b/setup.py @@ -23,14 +23,11 @@ ] if sys.version_info.major == 2: - install_requires.append('futures >= 3.0.5') + install_requires.extend(['futures >= 3.0.5', 'subprocess32']) if sys.version_info[:2] < (3, 5): install_requires.append('typing >= 3.5.2') -if sys.version_info[:2] < (3, 3): - install_requires.append('timeout_decorator >= 0.4.0') - setup(name='cwltest', version='1.0', description='Common workflow language testing framework', From b5096ab907c4a464622b6c6d7adb329a8f19190c Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 8 Feb 2018 17:26:52 +0100 Subject: [PATCH 3/4] use mypy compat verson of version check --- cwltest/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cwltest/__init__.py b/cwltest/__init__.py index a7a498c..9eda222 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -32,7 +32,7 @@ UNSUPPORTED_FEATURE = 33 DEFAULT_TIMEOUT = 900 # 15 minutes -if sys.version_info.major < 3: +if sys.version_info < (3, 0): import subprocess32 as subprocess else: import subprocess From a196df322f808a4104b5e09c9f60b491db20f0cb Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 8 Feb 2018 18:29:27 +0100 Subject: [PATCH 4/4] include subprocess32 stubs --- typeshed/2.7/subprocess32.pyi | 53 +++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 typeshed/2.7/subprocess32.pyi diff --git a/typeshed/2.7/subprocess32.pyi b/typeshed/2.7/subprocess32.pyi new file mode 100644 index 0000000..e7e8e58 --- /dev/null +++ b/typeshed/2.7/subprocess32.pyi @@ -0,0 +1,53 @@ +# Stubs for subprocess32 (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional + +class CalledProcessError(Exception): + returncode: Any = ... + cmd: Any = ... + output: Any = ... + def __init__(self, returncode, cmd, output: Optional[Any] = ...) -> None: ... + +class TimeoutExpired(Exception): + cmd: Any = ... + timeout: Any = ... + output: Any = ... + def __init__(self, cmd, timeout, output: Optional[Any] = ...) -> None: ... + +class STARTUPINFO: + dwFlags: int = ... + hStdInput: Any = ... + hStdOutput: Any = ... + hStdError: Any = ... + wShowWindow: int = ... + +class pywintypes: + error: Any = ... + +PIPE: int +STDOUT: int + +def call(*popenargs, **kwargs): ... +def check_call(*popenargs, **kwargs): ... +def check_output(*popenargs, **kwargs): ... + +class Popen: + args: Any = ... + stdin: Any = ... + stdout: Any = ... + stderr: Any = ... + pid: Any = ... + returncode: Any = ... + universal_newlines: Any = ... + def __init__(self, args, bufsize: int = ..., executable: Optional[Any] = ..., stdin: Optional[Any] = ..., stdout: Optional[Any] = ..., stderr: Optional[Any] = ..., preexec_fn: Optional[Any] = ..., close_fds: Any = ..., shell: bool = ..., cwd: Optional[Any] = ..., env: Optional[Any] = ..., universal_newlines: bool = ..., startupinfo: Optional[Any] = ..., creationflags: int = ..., restore_signals: bool = ..., start_new_session: bool = ..., pass_fds: Any = ...) -> None: ... + def __enter__(self): ... + def __exit__(self, type, value, traceback): ... + def __del__(self, _maxint: Any = ..., _active: Any = ...): ... + def communicate(self, input: Optional[Any] = ..., timeout: Optional[Any] = ...): ... + def poll(self) -> int: ... + def wait(self, timeout: Optional[Any] = ..., endtime: Optional[Any] = ...): ... + def send_signal(self, sig): ... + def terminate(self): ... + def kill(self): ...