From 2c4215df9a9a5d9fbf1b8abc7bd9ed1ced782ba7 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 3 May 2019 18:01:58 -0400 Subject: [PATCH 1/4] Report when a required test returns unsupported feature --- cwltest/__init__.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cwltest/__init__.py b/cwltest/__init__.py index 727074d..fbf9dd2 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -126,13 +126,16 @@ def run_test(args, # type: argparse.Namespace _logger.error(outstr) _logger.error(outerr) except subprocess.CalledProcessError as err: - if err.returncode == UNSUPPORTED_FEATURE: + if err.returncode == UNSUPPORTED_FEATURE and REQUIRED not in test.get("tags", []): return TestResult(UNSUPPORTED_FEATURE, outstr, outerr, duration, args.classname) if test.get("should_fail", False): return TestResult(0, outstr, outerr, duration, args.classname) _logger.error(u"""Test %i failed: %s""", test_number, " ".join([quote(tc) for tc in test_command])) _logger.error(test.get("doc")) - _logger.error(u"Returned non-zero") + if err.returncode == UNSUPPORTED_FEATURE: + _logger.error(u"Does not support required feature") + else: + _logger.error(u"Returned non-zero") _logger.error(outerr) return TestResult(1, outstr, outerr, duration, args.classname, str(err)) except (yamlscanner.ScannerError, TypeError) as err: @@ -273,9 +276,9 @@ def main(): # type: () -> int if args.l: for i, t in enumerate(tests): if t.get("short_name"): - print(u"[%i] %s: %s" % (i + 1, t["short_name"], t["doc"].strip())) + print(u"[%i] %s: %s" % (i + 1, t["short_name"], t.get("doc", "").strip())) else: - print(u"[%i] %s" % (i + 1, t["doc"].strip())) + print(u"[%i] %s" % (i + 1, t.get("doc", "").strip())) return 0 From 963ef48f7d6e0a61dc5865953f5f49eb8ddbfe8d Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 3 May 2019 18:26:52 -0400 Subject: [PATCH 2/4] Fix unit test for required unsupported features --- cwltest/__init__.py | 2 +- cwltest/tests/test_categories.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cwltest/__init__.py b/cwltest/__init__.py index fbf9dd2..49993b8 100755 --- a/cwltest/__init__.py +++ b/cwltest/__init__.py @@ -126,7 +126,7 @@ def run_test(args, # type: argparse.Namespace _logger.error(outstr) _logger.error(outerr) except subprocess.CalledProcessError as err: - if err.returncode == UNSUPPORTED_FEATURE and REQUIRED not in test.get("tags", []): + if err.returncode == UNSUPPORTED_FEATURE and REQUIRED not in test.get("tags", ["required"]): return TestResult(UNSUPPORTED_FEATURE, outstr, outerr, duration, args.classname) if test.get("should_fail", False): return TestResult(0, outstr, outerr, duration, args.classname) diff --git a/cwltest/tests/test_categories.py b/cwltest/tests/test_categories.py index fbfa02a..1b1e718 100644 --- a/cwltest/tests/test_categories.py +++ b/cwltest/tests/test_categories.py @@ -1,5 +1,5 @@ import unittest - +import re import os from os import linesep as n @@ -13,11 +13,22 @@ def test_unsupported_with_required_tests(self): args = ["--test", get_data("tests/test-data/required-unsupported.yml")] error_code, stdout, stderr = run_with_mock_cwl_runner(args) self.assertEqual(error_code, 1) + print(stderr) + stderr = re.sub(r' --outdir=[^ ]*', '', stderr) self.assertEqual( - "Test [1/2] Required test that is unsupported (without tags){n}{n}" - "Test [2/2] Required test that is unsupported (with tags){n}{n}" - "0 tests passed, 2 failures, 0 unsupported " - "features{n}".format(n=n), stderr) + "Test [1/2] Required test that is unsupported (without tags){n}" + "{n}" + "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0/cat-job.json{n}" + "Required test that is unsupported (without tags){n}" + "Does not support required feature{n}" + "{n}" + "Test [2/2] Required test that is unsupported (with tags){n}" + "{n}" + "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0/cat-job.json{n}" + "Required test that is unsupported (with tags){n}" + "Does not support required feature{n}" + "{n}" + "0 tests passed, 2 failures, 0 unsupported features{n}".format(n=n), stderr) def test_unsupported_with_optional_tests(self): args = ["--test", get_data("tests/test-data/optional-unsupported.yml")] From 3c05901e7a60f09bfc1fc729eb0d976b38c4748b Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Fri, 3 May 2019 18:43:52 -0400 Subject: [PATCH 3/4] Cross-platform tests --- cwltest/tests/test_categories.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cwltest/tests/test_categories.py b/cwltest/tests/test_categories.py index 1b1e718..ef3633c 100644 --- a/cwltest/tests/test_categories.py +++ b/cwltest/tests/test_categories.py @@ -2,6 +2,7 @@ import re import os from os import linesep as n +from os import sep as p from .util import run_with_mock_cwl_runner, get_data import xml.etree.ElementTree as ET @@ -14,21 +15,21 @@ def test_unsupported_with_required_tests(self): error_code, stdout, stderr = run_with_mock_cwl_runner(args) self.assertEqual(error_code, 1) print(stderr) - stderr = re.sub(r' --outdir=[^ ]*', '', stderr) + stderr = re.sub(r" '?--outdir=[^ ]*", '', stderr) self.assertEqual( "Test [1/2] Required test that is unsupported (without tags){n}" "{n}" - "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0/cat-job.json{n}" + "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0{p}cat-job.json{n}" "Required test that is unsupported (without tags){n}" "Does not support required feature{n}" "{n}" "Test [2/2] Required test that is unsupported (with tags){n}" "{n}" - "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0/cat-job.json{n}" + "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0{p}cat-job.json{n}" "Required test that is unsupported (with tags){n}" "Does not support required feature{n}" "{n}" - "0 tests passed, 2 failures, 0 unsupported features{n}".format(n=n), stderr) + "0 tests passed, 2 failures, 0 unsupported features{n}".format(n=n, p=p), stderr) def test_unsupported_with_optional_tests(self): args = ["--test", get_data("tests/test-data/optional-unsupported.yml")] From 569296da6176751742b69e8aaa0b591b48877451 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Mon, 6 May 2019 10:10:17 -0400 Subject: [PATCH 4/4] Windows testing workaround --- cwltest/tests/test_categories.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cwltest/tests/test_categories.py b/cwltest/tests/test_categories.py index ef3633c..61764f1 100644 --- a/cwltest/tests/test_categories.py +++ b/cwltest/tests/test_categories.py @@ -16,20 +16,24 @@ def test_unsupported_with_required_tests(self): self.assertEqual(error_code, 1) print(stderr) stderr = re.sub(r" '?--outdir=[^ ]*", '', stderr) + if os.name == 'nt': + q = "'" + else: + q = "" self.assertEqual( "Test [1/2] Required test that is unsupported (without tags){n}" "{n}" - "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0{p}cat-job.json{n}" + "Test 1 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}" "Required test that is unsupported (without tags){n}" "Does not support required feature{n}" "{n}" "Test [2/2] Required test that is unsupported (with tags){n}" "{n}" - "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl v1.0{p}cat-job.json{n}" + "Test 2 failed: mock-cwl-runner --quiet return-unsupported.cwl {q}v1.0{p}cat-job.json{q}{n}" "Required test that is unsupported (with tags){n}" "Does not support required feature{n}" "{n}" - "0 tests passed, 2 failures, 0 unsupported features{n}".format(n=n, p=p), stderr) + "0 tests passed, 2 failures, 0 unsupported features{n}".format(n=n, p=p, q=q), stderr) def test_unsupported_with_optional_tests(self): args = ["--test", get_data("tests/test-data/optional-unsupported.yml")]