diff --git a/cwltest/__init__.py b/cwltest/__init__.py index d3a35a5..9eda222 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -3,66 +3,46 @@ from __future__ import absolute_import from __future__ import print_function -from six.moves import range -from six.moves import zip - import argparse import json +import logging import os -import subprocess -import sys +import pipes import shutil +import sys import tempfile +import threading +import time + import junit_xml import ruamel.yaml as yaml import ruamel.yaml.scanner as yamlscanner -import pipes -import logging import schema_salad.ref_resolver -import time -import threading from concurrent.futures import ThreadPoolExecutor -from typing import Any, Dict, List, Text +from six.moves import range +from six.moves import zip +from typing import Any, Dict, List -from cwltest.utils import compare, CompareFail +from cwltest.utils import compare, CompareFail, TestResult _logger = logging.getLogger("cwltest") _logger.addHandler(logging.StreamHandler()) _logger.setLevel(logging.INFO) UNSUPPORTED_FEATURE = 33 -RUNTIME = sys.version_info.major - - -class TestResult(object): - - """Encapsulate relevant test result data.""" - - def __init__(self, return_code, standard_output, error_output, duration, classname, message=''): - # type: (int, Text, Text, float, Text, str) -> None - self.return_code = return_code - self.standard_output = standard_output - self.error_output = error_output - self.duration = duration - self.message = message - self.classname = classname - - def create_test_case(self, test): - # type: (Dict[Text, Any]) -> junit_xml.TestCase - doc = test.get(u'doc', 'N/A').strip() - case = junit_xml.TestCase( - doc, elapsed_sec=self.duration, classname=self.classname, - stdout=self.standard_output, stderr=self.error_output, - ) - if self.return_code > 0: - case.failure_message = self.message - return case +DEFAULT_TIMEOUT = 900 # 15 minutes +if sys.version_info < (3, 0): + import subprocess32 as subprocess +else: + import subprocess templock = threading.Lock() -def run_test(args, i, tests): # type: (argparse.Namespace, int, List[Dict[str, str]]) -> TestResult +def run_test(args, i, tests, timeout): + # type: (argparse.Namespace, int, List[Dict[str, str]], int) -> TestResult + global templock out = {} # type: Dict[str,Any] @@ -105,7 +85,7 @@ def run_test(args, i, tests): # type: (argparse.Namespace, int, List[Dict[str, start_time = time.time() stderr = subprocess.PIPE if not args.verbose else None process = subprocess.Popen(test_command, stdout=subprocess.PIPE, stderr=stderr) - outstr, outerr = [var.decode('utf-8') for var in process.communicate()] + outstr, outerr = [var.decode('utf-8') for var in process.communicate(timeout=timeout)] return_code = process.poll() duration = time.time() - start_time if return_code: @@ -135,6 +115,10 @@ def run_test(args, i, tests): # type: (argparse.Namespace, int, List[Dict[str, except KeyboardInterrupt: _logger.error(u"""Test interrupted: %s""", " ".join([pipes.quote(tc) for tc in test_command])) raise + except subprocess.TimeoutExpired: + _logger.error(u"""Test timed out: %s""", " ".join([pipes.quote(tc) for tc in test_command])) + _logger.error(t.get("doc")) + return TestResult(2, outstr, outerr, timeout, args.classname, "Test timed out") fail_message = '' @@ -177,6 +161,9 @@ def main(): # type: () -> int "(defaults to one).") parser.add_argument("--verbose", action="store_true", help="More verbose output during test run.") parser.add_argument("--classname", type=str, default="", help="Specify classname for the Test Suite.") + parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="Time of execution in seconds after " + "which the test will be skipped." + "Defaults to 900 sec (15 minutes)") args = parser.parse_args() if '--' in args.args: @@ -229,7 +216,7 @@ def main(): # type: () -> int total = 0 with ThreadPoolExecutor(max_workers=args.j) as executor: - jobs = [executor.submit(run_test, args, i, tests) + jobs = [executor.submit(run_test, args, i, tests, args.timeout) for i in ntest] try: for i, job in zip(ntest, jobs): diff --git a/cwltest/utils.py b/cwltest/utils.py index 9808dc6..66da97d 100644 --- a/cwltest/utils.py +++ b/cwltest/utils.py @@ -1,9 +1,36 @@ import json -from typing import Any, Dict, Set + +import junit_xml +from typing import Any, Dict, Set, Text from six.moves import range +class TestResult(object): + + """Encapsulate relevant test result data.""" + + def __init__(self, return_code, standard_output, error_output, duration, classname, message=''): + # type: (int, Text, Text, float, Text, str) -> None + self.return_code = return_code + self.standard_output = standard_output + self.error_output = error_output + self.duration = duration + self.message = message + self.classname = classname + + def create_test_case(self, test): + # type: (Dict[Text, Any]) -> junit_xml.TestCase + doc = test.get(u'doc', 'N/A').strip() + case = junit_xml.TestCase( + doc, elapsed_sec=self.duration, classname=self.classname, + stdout=self.standard_output, stderr=self.error_output, + ) + if self.return_code > 0: + case.failure_message = self.message + return case + + class CompareFail(Exception): @classmethod diff --git a/requirements.txt b/requirements.txt index 7ca9fd3..600a7db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ schema-salad >= 1.14 typing>=3.6,<3.7; python_version < '3.5' futures >= 3.0.5; python_version == '2.7' +subprocess32; python_version < '3' junit-xml >= 1.7 + diff --git a/setup.py b/setup.py index 81cab1b..2b16faf 100755 --- a/setup.py +++ b/setup.py @@ -23,12 +23,11 @@ ] if sys.version_info.major == 2: - install_requires.append('futures >= 3.0.5') + install_requires.extend(['futures >= 3.0.5', 'subprocess32']) if sys.version_info[:2] < (3, 5): install_requires.append('typing >= 3.5.2') - setup(name='cwltest', version='1.0', description='Common workflow language testing framework', diff --git a/typeshed/2.7/subprocess32.pyi b/typeshed/2.7/subprocess32.pyi new file mode 100644 index 0000000..e7e8e58 --- /dev/null +++ b/typeshed/2.7/subprocess32.pyi @@ -0,0 +1,53 @@ +# Stubs for subprocess32 (Python 2) +# +# NOTE: This dynamically typed stub was automatically generated by stubgen. + +from typing import Any, Optional + +class CalledProcessError(Exception): + returncode: Any = ... + cmd: Any = ... + output: Any = ... + def __init__(self, returncode, cmd, output: Optional[Any] = ...) -> None: ... + +class TimeoutExpired(Exception): + cmd: Any = ... + timeout: Any = ... + output: Any = ... + def __init__(self, cmd, timeout, output: Optional[Any] = ...) -> None: ... + +class STARTUPINFO: + dwFlags: int = ... + hStdInput: Any = ... + hStdOutput: Any = ... + hStdError: Any = ... + wShowWindow: int = ... + +class pywintypes: + error: Any = ... + +PIPE: int +STDOUT: int + +def call(*popenargs, **kwargs): ... +def check_call(*popenargs, **kwargs): ... +def check_output(*popenargs, **kwargs): ... + +class Popen: + args: Any = ... + stdin: Any = ... + stdout: Any = ... + stderr: Any = ... + pid: Any = ... + returncode: Any = ... + universal_newlines: Any = ... + def __init__(self, args, bufsize: int = ..., executable: Optional[Any] = ..., stdin: Optional[Any] = ..., stdout: Optional[Any] = ..., stderr: Optional[Any] = ..., preexec_fn: Optional[Any] = ..., close_fds: Any = ..., shell: bool = ..., cwd: Optional[Any] = ..., env: Optional[Any] = ..., universal_newlines: bool = ..., startupinfo: Optional[Any] = ..., creationflags: int = ..., restore_signals: bool = ..., start_new_session: bool = ..., pass_fds: Any = ...) -> None: ... + def __enter__(self): ... + def __exit__(self, type, value, traceback): ... + def __del__(self, _maxint: Any = ..., _active: Any = ...): ... + def communicate(self, input: Optional[Any] = ..., timeout: Optional[Any] = ...): ... + def poll(self) -> int: ... + def wait(self, timeout: Optional[Any] = ..., endtime: Optional[Any] = ...): ... + def send_signal(self, sig): ... + def terminate(self): ... + def kill(self): ...