Merge branch 'master' into check-contents

Makefile

@@ -147,17 +147,28 @@ list-author-emails:
 	@echo 'name, E-Mail Address'
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
-mypy: ${PYSOURCES}
+mypy2: ${PYSOURCES}
 	rm -Rf typeshed/2.7/ruamel/yaml
 	ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
 		typeshed/2.7/ruamel/yaml
 	rm -Rf typeshed/2.7/schema_salad
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
 		typeshed/2.7/schema_salad
-	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
+	MYPYPATH=typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
 		 --warn-redundant-casts --warn-unused-ignores \
 		 ${MODULE}
 
+mypy3: ${PYSOURCES}
+	rm -Rf typeshed/2and3/ruamel/yaml
+	ln -s $(shell python3 -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
+		typeshed/2and3/ruamel/yaml
+	rm -Rf typeshed/2and3/schema_salad
+	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
+		typeshed/2and3/schema_salad
+	MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+		 --warn-redundant-casts \
+		 ${MODULE}
+
 release: FORCE
 	./release-test.sh
 	. testenv2/bin/activate && \

cwltest/__init__.py

@@ -3,66 +3,74 @@
 from __future__ import absolute_import
 from __future__ import print_function
 
-from six.moves import range
-from six.moves import zip
-
 import argparse
 import json
+import logging
 import os
-import subprocess
-import sys
+import pipes
 import shutil
+import sys
 import tempfile
+import threading
+import time
+
 import junit_xml
 import ruamel.yaml as yaml
 import ruamel.yaml.scanner as yamlscanner
-import pipes
-import logging
 import schema_salad.ref_resolver
-import time
-import threading
 from concurrent.futures import ThreadPoolExecutor
+from six.moves import range
+from six.moves import zip
 from typing import Any, Dict, List, Text
 
-from cwltest.utils import compare, CompareFail
+from cwltest.utils import compare, CompareFail, TestResult
 
 _logger = logging.getLogger("cwltest")
 _logger.addHandler(logging.StreamHandler())
 _logger.setLevel(logging.INFO)
 
 UNSUPPORTED_FEATURE = 33
-RUNTIME = sys.version_info.major
+DEFAULT_TIMEOUT = 900  # 15 minutes
 
+if sys.version_info < (3, 0):
+    import subprocess32 as subprocess
+else:
+    import subprocess
 
-class TestResult(object):
-
-    """Encapsulate relevant test result data."""
+templock = threading.Lock()
 
-    def __init__(self, return_code, standard_output, error_output, duration, classname, message=''):
-        # type: (int, Text, Text, float, Text, str) -> None
-        self.return_code = return_code
-        self.standard_output = standard_output
-        self.error_output = error_output
-        self.duration = duration
-        self.message = message
-        self.classname = classname
 
-    def create_test_case(self, test):
-        # type: (Dict[Text, Any]) -> junit_xml.TestCase
-        doc = test.get(u'doc', 'N/A').strip()
-        case = junit_xml.TestCase(
-            doc, elapsed_sec=self.duration, classname=self.classname,
-            stdout=self.standard_output, stderr=self.error_output,
-        )
-        if self.return_code > 0:
-            case.failure_message = self.message
-        return case
+def prepare_test_command(args, i, tests):
+    # type: (argparse.Namespace, int, List[Dict[str, str]]) -> List[str]
+    t = tests[i]
+    test_command = [args.tool]
+    test_command.extend(args.args)
 
+    # Add additional arguments given in test case
+    if args.testargs is not None:
+        for testarg in args.testargs:
+            (test_case_name, prefix) = testarg.split('==')
+            if test_case_name in t:
+                test_command.extend([prefix, t[test_case_name]])
+
+    # Add prefixes if running on MacOSX so that boot2docker writes to /Users
+    with templock:
+        if 'darwin' in sys.platform and args.tool == 'cwltool':
+            outdir = tempfile.mkdtemp(prefix=os.path.abspath(os.path.curdir))
+            test_command.extend(["--tmp-outdir-prefix={}".format(outdir), "--tmpdir-prefix={}".format(outdir)])
+        else:
+            outdir = tempfile.mkdtemp()
+    test_command.extend(["--outdir={}".format(outdir),
+                         "--quiet",
+                         t["tool"]])
+    if t.get("job"):
+        test_command.append(t["job"])
+    return test_command
 
-templock = threading.Lock()
 
+def run_test(args, i, tests, timeout):
+    # type: (argparse.Namespace, int, List[Dict[str, str]], int) -> TestResult
 
-def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str, str]]) -> TestResult
     global templock
 
     out = {}  # type: Dict[str,Any]
@@ -76,36 +84,15 @@ def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str,
     else:
         suffix = "\n"
     try:
-        test_command = [args.tool]
-        test_command.extend(args.args)
-
-        # Add additional arguments given in test case
-        if args.testargs is not None:
-            for testarg in args.testargs:
-                (test_case_name, prefix) = testarg.split('==')
-                if test_case_name in t:
-                    test_command.extend([prefix, t[test_case_name]])
-
-        # Add prefixes if running on MacOSX so that boot2docker writes to /Users
-        with templock:
-            if 'darwin' in sys.platform and args.tool == 'cwltool':
-                outdir = tempfile.mkdtemp(prefix=os.path.abspath(os.path.curdir))
-                test_command.extend(["--tmp-outdir-prefix={}".format(outdir), "--tmpdir-prefix={}".format(outdir)])
-            else:
-                outdir = tempfile.mkdtemp()
-        test_command.extend(["--outdir={}".format(outdir),
-                             "--quiet",
-                             t["tool"]])
-        if t.get("job"):
-            test_command.append(t["job"])
+        test_command = prepare_test_command(args, i, tests)
 
         sys.stderr.write("%sTest [%i/%i] %s\n" % (prefix, i + 1, len(tests), suffix))
         sys.stderr.flush()
 
         start_time = time.time()
         stderr = subprocess.PIPE if not args.verbose else None
         process = subprocess.Popen(test_command, stdout=subprocess.PIPE, stderr=stderr)
-        outstr, outerr = [var.decode('utf-8') for var in process.communicate()]
+        outstr, outerr = [var.decode('utf-8') for var in process.communicate(timeout=timeout)]
         return_code = process.poll()
         duration = time.time() - start_time
         if return_code:
@@ -135,6 +122,10 @@ def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str,
     except KeyboardInterrupt:
         _logger.error(u"""Test interrupted: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
         raise
+    except subprocess.TimeoutExpired:
+        _logger.error(u"""Test timed out: %s""", " ".join([pipes.quote(tc) for tc in test_command]))
+        _logger.error(t.get("doc"))
+        return TestResult(2, outstr, outerr, timeout, args.classname, "Test timed out")
 
     fail_message = ''
 
@@ -158,8 +149,7 @@ def run_test(args, i, tests):  # type: (argparse.Namespace, int, List[Dict[str,
     return TestResult((1 if fail_message else 0), outstr, outerr, duration, args.classname, fail_message)
 
 
-def main():  # type: () -> int
-
+def arg_parser():  # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(description='Compliance tests for cwltool')
     parser.add_argument("--test", type=str, help="YAML file describing test cases", required=True)
     parser.add_argument("--basedir", type=str, help="Basedir to use for tests", default=".")
@@ -177,8 +167,15 @@ def main():  # type: () -> int
                                                         "(defaults to one).")
     parser.add_argument("--verbose", action="store_true", help="More verbose output during test run.")
     parser.add_argument("--classname", type=str, default="", help="Specify classname for the Test Suite.")
+    parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT, help="Time of execution in seconds after "
+                                                                             "which the test will be skipped."
+                                                                             "Defaults to 900 sec (15 minutes)")
+    return parser
+
+
+def main():  # type: () -> int
 
-    args = parser.parse_args()
+    args = arg_parser().parse_args(sys.argv[1:])
     if '--' in args.args:
         args.args.remove('--')
 
@@ -187,7 +184,7 @@ def main():  # type: () -> int
         args.testargs = [testarg for testarg in args.testargs if testarg.count('==') == 1]
 
     if not args.test:
-        parser.print_help()
+        arg_parser().print_help()
         return 1
 
     with open(args.test) as f:
@@ -229,7 +226,7 @@ def main():  # type: () -> int
 
     total = 0
     with ThreadPoolExecutor(max_workers=args.j) as executor:
-        jobs = [executor.submit(run_test, args, i, tests)
+        jobs = [executor.submit(run_test, args, i, tests, args.timeout)
                 for i in ntest]
         try:
             for i, job in zip(ntest, jobs):

cwltest/utils.py

@@ -1,9 +1,36 @@
 import json
-from typing import Any, Dict, Set
+
+import junit_xml
+from typing import Any, Dict, Set, Text
 
 from six.moves import range
 
 
+class TestResult(object):
+
+    """Encapsulate relevant test result data."""
+
+    def __init__(self, return_code, standard_output, error_output, duration, classname, message=''):
+        # type: (int, Text, Text, float, Text, str) -> None
+        self.return_code = return_code
+        self.standard_output = standard_output
+        self.error_output = error_output
+        self.duration = duration
+        self.message = message
+        self.classname = classname
+
+    def create_test_case(self, test):
+        # type: (Dict[Text, Any]) -> junit_xml.TestCase
+        doc = test.get(u'doc', 'N/A').strip()
+        case = junit_xml.TestCase(
+            doc, elapsed_sec=self.duration, classname=self.classname,
+            stdout=self.standard_output, stderr=self.error_output,
+        )
+        if self.return_code > 0:
+            case.failure_message = self.message
+        return case
+
+
 class CompareFail(Exception):
 
     @classmethod

mypy.ini

@@ -1,2 +1,4 @@
+[mypy]
+
 [mypy-ruamel.*]
-ignore_errors = True
+ignore_errors = True

requirements.txt

@@ -1,4 +1,6 @@
 schema-salad >= 1.14
 typing>=3.6,<3.7; python_version < '3.5'
 futures >= 3.0.5; python_version == '2.7'
+subprocess32; python_version < '3'
 junit-xml >= 1.7
+

setup.py

@@ -23,12 +23,11 @@
 ]
 
 if sys.version_info.major == 2:
-    install_requires.append('futures >= 3.0.5')
+    install_requires.extend(['futures >= 3.0.5', 'subprocess32'])
 
 if sys.version_info[:2] < (3, 5):
     install_requires.append('typing >= 3.5.2')
 
-
 setup(name='cwltest',
       version='1.0',
       description='Common workflow language testing framework',
@@ -40,6 +39,7 @@
       license='Apache 2.0',
       packages=["cwltest"],
       install_requires=install_requires,
+      test_suite='tests',
       tests_require=[],
       entry_points={
           'console_scripts': ["cwltest=cwltest:main"]

tests/test_argparse.py

@@ -0,0 +1,18 @@
+import unittest
+from cwltest import arg_parser
+
+class TestArgparse(unittest.TestCase):
+    def setUp(self):
+        self.parser = arg_parser()
+
+    def test_arg(self):
+        parsed = self.parser.parse_args(['--test', 'test_name','-n','52','--tool','cwltool','-j','4'])
+        self.assertEqual(parsed.test, 'test_name')
+        self.assertEqual(parsed.n, '52')
+        self.assertEqual(parsed.tool, 'cwltool')
+        self.assertEqual(parsed.j, 4)
+
+
+if __name__ == '__main__':
+    unittest.main()
+

tests/test_compare.py

@@ -0,0 +1,28 @@
+import unittest
+from cwltest import CompareFail
+from cwltest.utils import compare_file
+
+
+class TestCompareFile(unittest.TestCase):
+
+    def test_general(self):
+        expected = {
+            "location": "cores.txt",
+            "size": 2,
+            "class": "File",
+            "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a"
+         }
+
+        actual = {
+            "basename": "cores.txt",
+            "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a",
+            "class": "File",
+            "location": "file:///var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
+            "path": "/var/folders/8x/2df05_7j20j6r8y81w4qf43r0000gn/T/tmpG0EkrS/cores.txt",
+            "size": 2
+        }
+
+        try:
+            compare_file(expected, actual)
+        except CompareFail:
+            self.fail("File comparison failed unexpectedly")

tox.ini

@@ -1,24 +1,31 @@
 [tox]
 #envlist = py35-lint,py34-lint,py33-lint,py27-lint,py35-unit,py34-unit,py33-unit,py27-unit
-envlist = py27-lint, py27-unit, py35-mypy
+envlist = py27-lint, py27-unit, py35-mypy{2,3}
 skipsdist = True
 
 [travis]
 os =
-  linux: py{27,35}-{lint,unit}, py35-mypy
+  linux: py{27,35}-{lint,unit}, py35-mypy{2,3}
   osx: py{27}-{lint,unit}
 python =
   2.7: py27
-  3.5: py35-mypy
+  3.5: py35-mypy{2,3}
 
 [testenv]
 deps = -rrequirements.txt
 
-[testenv:py35-mypy]
-commands = make mypy
+[testenv:py35-mypy2]
+commands = make mypy2
 whitelist_externals = make
 deps =
-  mypy==0.520
+  mypy==0.560
+  -rrequirements.txt
+
+[testenv:py35-mypy3]
+commands = make mypy3
+whitelist_externals = make
+deps =
+  mypy==0.560
   -rrequirements.txt
 
 [testenv:py35-lint]