From 06b704a27237a476ea2a5318b564e29575fba111 Mon Sep 17 00:00:00 2001 From: "Michael R. Crusoe" Date: Thu, 13 Feb 2020 09:51:52 +0100 Subject: [PATCH] drop Python2 support --- .travis.yml | 8 +--- Makefile | 58 ++++++++++--------------- README.rst | 2 +- cwlupgrader/main.py | 97 +++++++++++++++++++----------------------- setup.cfg | 3 -- setup.py | 8 ++-- tests/test_complete.py | 2 + tests/util.py | 8 ++-- tox.ini | 31 ++++++-------- 9 files changed, 93 insertions(+), 124 deletions(-) diff --git a/.travis.yml b/.travis.yml index 133b6256..6aac07be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ -dist: trusty -sudo: required +dist: xenial services: - docker @@ -15,13 +14,10 @@ install: - pip install tox-travis jobs: include: - - python: "2.7" - python: "3.5" - python: "3.6" - python: "3.7" - dist: xenial - #- python: "3.8-dev" - #dist: xenial + - python: "3.8" script: tox branches: diff --git a/Makefile b/Makefile index d35b9937..5589074f 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ # # Contact: common-workflow-language@googlegroups.com -# make pycodestyle to check for basic Python code compliance +# make format to fix most python formatting errors # make pylint to check Python code for enhanced compliance including naming # and documentation # make coverage-report to check coverage of the python scripts by the tests @@ -25,15 +25,16 @@ MODULE=cwlupgrader # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard cwlupgrader/**.py tests/*.py) setup.py -DEVPKGS=pycodestyle diff_cover black pylint coverage pydocstyle \ - flake8-bugbear pytest isort mock flake8 -DEBDEVPKGS=pylint python-coverage pydocstyle sloccount \ - python-flake8 python-mock shellcheck -VERSION=$(shell git describe --tags --dirty | sed s/v//) +DEVPKGS=diff_cover black pylint coverage pep257 pytest-xdist \ + flake8-bugbear pytest isort flake8 +DEBDEVPKGS=pylint python3-coverage sloccount \ + python3-flake8 shellcheck +VERSION=1.0.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \ + --max-count=1 --format=format:%cI`) ## all : default task all: - ./setup.py develop + pip install -e . ## help : print this help message and exit help: Makefile @@ -69,39 +70,29 @@ clean: FORCE sort_imports: isort ${MODULE}/*.py tests/*.py setup.py -## pycodestyle : check Python code style -pycodestyle: $(PYSOURCES) - pycodestyle --exclude=_version.py --show-source --show-pep8 $^ || true - -pycodestyle_report.txt: $(PYSOURCES) - pycodestyle --exclude=_version.py $^ > $@ || true - -diff_pycodestyle_report: pycodestyle_report.txt - diff-quality --violations=pycodestyle $^ - pep257: pydocstyle ## pydocstyle : check Python code style pydocstyle: $(PYSOURCES) - pydocstyle --ignore=D100,D101,D102,D103 $^ || true + pydocstyle --add-ignore=D100,D101,D102,D103 $^ || true pydocstyle_report.txt: $(PYSOURCES) - pydocstyle setup.py $^ > pydocstyle_report.txt 2>&1 || true + pydocstyle setup.py $^ > $@ 2>&1 || true diff_pydocstyle_report: pydocstyle_report.txt - diff-quality --violations=pycodestyle $^ + diff-quality --violations=pycodestyle --fail-under=100 $^ ## format : check/fix all code indentation and formatting (runs black) format: - black --target-version py27 setup.py cwlupgrader + black setup.py cwlupgrader ## pylint : run static code analysis on Python code pylint: $(PYSOURCES) pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ - $^ || true + $^ -j0|| true pylint_report.txt: ${PYSOURCES} pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ - $^ > $@ || true + $^ -j0> $@ || true diff_pylint_report: pylint_report.txt diff-quality --violations=pylint pylint_report.txt @@ -148,19 +139,14 @@ list-author-emails: @echo 'name, E-Mail Address' @git log --format='%aN,%aE' | sort -u | grep -v 'root' - -mypy2: ${PYSOURCES} - rm -Rf typeshed/2and3/ruamel/yaml - ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \ - typeshed/2and3/ruamel/yaml - MYPYPATH=$$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \ - --warn-redundant-casts \ - ${MODULE} - -mypy3: ${PYSOURCES} - rm -Rf typeshed/2and3/ruamel/yaml - ln -s $(shell python3 -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \ - typeshed/2and3/ruamel/yaml +mypy3: mypy +mypy: ${PYSOURCES} + if ! test -f $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \ + then \ + rm -Rf typeshed/2and3/ruamel/yaml ; \ + ln -s $(shell python3 -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \ + typeshed/2and3/ruamel/ ; \ + fi # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \ --warn-redundant-casts \ ${MODULE} diff --git a/README.rst b/README.rst index 667e2e68..da63fd94 100644 --- a/README.rst +++ b/README.rst @@ -8,7 +8,7 @@ version "draft-3" to "v1.0". It does not check for correctness of the input document, for that one can use the CWL reference implementation. -This is written and tested for Python 2.7, 3.4, and 3.5. +This is written and tested for Python 3.5, 3.6, 3.7, and 3.8. Install ------- diff --git a/cwlupgrader/main.py b/cwlupgrader/main.py index 093447d8..8211083d 100755 --- a/cwlupgrader/main.py +++ b/cwlupgrader/main.py @@ -1,22 +1,11 @@ #!/usr/bin/env python """Transforms draft-3 CWL documents into v1.0 as idiomatically as possible.""" -from __future__ import print_function - -try: - from collections.abs import Mapping, MutableMapping, MutableSequence, Sequence -except ImportError: - from collections import Mapping, MutableMapping, MutableSequence, Sequence -import sys import copy -from typing import ( - Any, - Dict, - List, - Optional, # pylint:disable=unused-import - Text, - Union, -) +import sys +from collections import MutableMapping, MutableSequence, Sequence +from typing import Any, Dict, List, Optional, Union + import ruamel.yaml from ruamel.yaml.comments import CommentedMap # for consistent sort order @@ -28,7 +17,7 @@ def main(args=None): # type: (Optional[List[str]]) -> int assert args is not None for path in args: with open(path) as entry: - document = ruamel.yaml.safe_load(entry) + document = ruamel.yaml.main.safe_load(entry) if "cwlVersion" in document: if ( document["cwlVersion"] == "cwl:draft-3" @@ -38,43 +27,42 @@ def main(args=None): # type: (Optional[List[str]]) -> int elif document["cwlVersion"] == "v1.0": document = v1_0_to_v1_1(document) ruamel.yaml.scalarstring.walk_tree(document) - print(ruamel.yaml.round_trip_dump(document, default_flow_style=False)) + print(ruamel.yaml.main.round_trip_dump(document, default_flow_style=False)) return 0 -def v1_0_to_v1_1(document): # type: (Dict[Text, Any]) -> Dict +def v1_0_to_v1_1(document): # type: (Dict[str, Any]) -> Dict[str, Any] """CWL v1.0.x to v1.1 transformation loop.""" # TODO: handle $import, see imported-hint.cwl schemadef-tool.cwl schemadef-wf.cwl _v1_0_to_v1_1(document) if isinstance(document, MutableMapping): for key, value in document.items(): - if isinstance(value, MutableMapping): + if isinstance(value, Dict): document[key] = _v1_0_to_v1_1(value) elif isinstance(value, list): for index, entry in enumerate(value): - if isinstance(entry, MutableMapping): + if isinstance(entry, Dict): value[index] = _v1_0_to_v1_1(entry) document["cwlVersion"] = "v1.1" return sort_v1_0(document) -def draft3_to_v1_0(document): # type: (Dict[Text, Any]) -> Dict +def draft3_to_v1_0(document): # type: (Dict[str, Any]) -> Dict[str, Any] """Transformation loop.""" _draft3_to_v1_0(document) if isinstance(document, MutableMapping): for key, value in document.items(): - if isinstance(value, MutableMapping): + if isinstance(value, Dict): document[key] = _draft3_to_v1_0(value) elif isinstance(value, list): for index, entry in enumerate(value): - if isinstance(entry, MutableMapping): + if isinstance(entry, Dict): value[index] = _draft3_to_v1_0(entry) document["cwlVersion"] = "v1.0" return sort_v1_0(document) -def _draft3_to_v1_0(document): - # type: (MutableMapping[Text, Any]) -> MutableMapping[Text, Any] +def _draft3_to_v1_0(document: Dict[str, Any]) -> Dict[str, Any]: """Inner loop for transforming draft-3 to v1.0.""" if "class" in document: if document["class"] == "Workflow": @@ -114,8 +102,7 @@ def _draft3_to_v1_0(document): } -def _v1_0_to_v1_1(document): - # type: (MutableMapping[Text, Any]) -> MutableMapping[Text, Any] +def _v1_0_to_v1_1(document: Dict[str, Any]) -> Dict[str, Any]: """Inner loop for transforming draft-3 to v1.0.""" if "class" in document: if document["class"] == "Workflow": @@ -126,7 +113,7 @@ def _v1_0_to_v1_1(document): if isinstance(steps, MutableSequence): for entry in steps: upgrade_v1_0_hints_and_reqs(entry) - if "run" in entry and isinstance(entry["run"], MutableMapping): + if "run" in entry and isinstance(entry["run"], Dict): process = entry["run"] _v1_0_to_v1_1(process) if "cwlVersion" in process: @@ -135,7 +122,7 @@ def _v1_0_to_v1_1(document): for step_name in steps: entry = steps[step_name] upgrade_v1_0_hints_and_reqs(entry) - if "run" in entry and isinstance(entry["run"], MutableMapping): + if "run" in entry and isinstance(entry["run"], Dict): process = entry["run"] _v1_0_to_v1_1(process) if "cwlVersion" in process: @@ -170,10 +157,10 @@ def _v1_0_to_v1_1(document): return document -def cleanup_v1_0_input_bindings(document): +def cleanup_v1_0_input_bindings(document: Dict[str, Any]) -> None: """In v1.1 Workflow or ExpressionTool level inputBindings are deprecated.""" - def cleanup(inp): + def cleanup(inp: Dict[str, Any]) -> None: """Serialize non loadContents fields and add that to the doc.""" if "inputBinding" in inp: bindings = inp["inputBinding"] @@ -194,10 +181,10 @@ def cleanup(inp): cleanup(inputs[input_name]) -def move_up_loadcontents(document): +def move_up_loadcontents(document: Dict[str, Any]) -> None: """'loadContents' is promoted up a level in CWL v1.1.""" - def cleanup(inp): + def cleanup(inp: Dict[str, Any]) -> None: """Move loadContents to the preferred location""" if "inputBinding" in inp: bindings = inp["inputBinding"] @@ -214,13 +201,15 @@ def cleanup(inp): cleanup(inputs[input_name]) -def upgrade_v1_0_hints_and_reqs(document): +def upgrade_v1_0_hints_and_reqs(document: Dict[str, Any]) -> None: for extra in ("requirements", "hints"): if extra in document: if isinstance(document[extra], MutableMapping): for req_name in document[extra]: if req_name in V1_0_TO_V1_1_REWRITE: - extra[V1_0_TO_V1_1_REWRITE[req_name]] = extra.pop(req_name) + document[extra][V1_0_TO_V1_1_REWRITE[req_name]] = document[ + extra + ].pop(req_name) elif isinstance(document[extra], MutableSequence): for entry in document[extra]: if entry["class"] in V1_0_TO_V1_1_REWRITE: @@ -228,11 +217,13 @@ def upgrade_v1_0_hints_and_reqs(document): else: raise Exception( "{} section must be either a list of dictionaries " - "or a dictionary of dictionaries!: {}" - ).format(extra, document[extra]) + "or a dictionary of dictionaries!: {}".format( + extra, document[extra] + ) + ) -def has_hint_or_req(document, name): +def has_hint_or_req(document: Dict[str, Any], name: str) -> bool: """Detects an existing named hint or requirement.""" for extra in ("requirements", "hints"): if extra in document: @@ -246,7 +237,7 @@ def has_hint_or_req(document, name): return False -def workflow_clean(document): # type: (MutableMapping[Text, Any]) -> None +def workflow_clean(document: Dict[str, Any]) -> None: """Transform draft-3 style Workflows to more idiomatic v1.0""" input_output_clean(document) hints_and_requirements_clean(document) @@ -289,7 +280,7 @@ def workflow_clean(document): # type: (MutableMapping[Text, Any]) -> None step["in"] = ins del step["inputs"] if "scatter" in step: - if isinstance(step["scatter"], (str, Text)) == 1: + if isinstance(step["scatter"], str) == 1: source = step["scatter"] if source.startswith(step_id): source = source[step_id_len:] @@ -311,7 +302,7 @@ def workflow_clean(document): # type: (MutableMapping[Text, Any]) -> None document["steps"] = new_steps -def input_output_clean(document): # type: (MutableMapping[Text, Any]) -> None +def input_output_clean(document: Dict[str, Any]) -> None: """Transform draft-3 style input/output listings into idiomatic v1.0.""" for param_type in ["inputs", "outputs"]: if param_type not in document: @@ -330,8 +321,7 @@ def input_output_clean(document): # type: (MutableMapping[Text, Any]) -> None document[param_type] = new_section -def hints_and_requirements_clean(document): - # type: (MutableMapping[Text, Any]) -> None +def hints_and_requirements_clean(document: Dict[str, Any]) -> None: """Transform draft-3 style hints/reqs into idiomatic v1.0 hints/reqs.""" for section in ["hints", "requirements"]: if section in document: @@ -353,14 +343,14 @@ def hints_and_requirements_clean(document): document[section] = new_section -def shorten_type(type_obj): # type: (List[Any]) -> Union[Text, List[Any]] +def shorten_type(type_obj: List[Any]) -> Union[str, List[Any]]: """Transform draft-3 style type declarations into idiomatic v1.0 types.""" - if isinstance(type_obj, (str, Text)) or not isinstance(type_obj, Sequence): + if isinstance(type_obj, str) or not isinstance(type_obj, Sequence): return type_obj - new_type = [] + new_type = [] # type: List[str] for entry in type_obj: # find arrays that we can shorten and do so - if isinstance(entry, Mapping): - if entry["type"] == "array" and isinstance(entry["items"], (str, Text)): + if isinstance(entry, Dict): + if entry["type"] == "array" and isinstance(entry["items"], str): entry = entry["items"] + "[]" elif entry["type"] == "enum": entry = sort_enum(entry) @@ -369,15 +359,14 @@ def shorten_type(type_obj): # type: (List[Any]) -> Union[Text, List[Any]] if "null" in new_type: type_copy = copy.deepcopy(new_type) type_copy.remove("null") - if isinstance(type_copy[0], (str, Text)): + if isinstance(type_copy[0], str): return type_copy[0] + "?" if len(new_type) == 1: return new_type[0] return new_type -def clean_secondary_files(document): - # type: (MutableMapping[Text, Any]) -> None +def clean_secondary_files(document: Dict[str, Any]) -> None: """Cleanup for secondaryFiles""" if "secondaryFiles" in document: for i, sfile in enumerate(document["secondaryFiles"]): @@ -387,7 +376,7 @@ def clean_secondary_files(document): ).replace(".path", ".location") -def sort_v1_0(document): # type: (Dict) -> Dict +def sort_v1_0(document: Dict[str, Any]) -> Dict[str, Any]: """Sort the sections of the CWL document in a more meaningful order.""" keyorder = [ "cwlVersion", @@ -418,7 +407,7 @@ def sort_v1_0(document): # type: (Dict) -> Dict ) -def sort_enum(enum): # type: (Mapping) -> Dict +def sort_enum(enum: Dict[str, Any]) -> Dict[str, Any]: """Sort the enum type definitions in a more meaningful order.""" keyorder = ["type", "name", "label", "symbols", "inputBinding"] return CommentedMap( @@ -429,7 +418,7 @@ def sort_enum(enum): # type: (Mapping) -> Dict ) -def sort_input_or_output(io_def): # type: (Dict) -> Dict +def sort_input_or_output(io_def: Dict[str, Any]) -> Dict[str, Any]: """Sort the input definitions in a more meaningful order.""" keyorder = [ "label", diff --git a/setup.cfg b/setup.cfg index 62e324a3..20a58b4d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,9 +3,6 @@ ignore = E124,E128,E129,E201,E202,E225,E226,E231,E265,E271,E302,E303,F401,E402,E [easy_install] -[bdist_wheel] -universal = 1 - [aliases] test=pytest diff --git a/setup.py b/setup.py index e2a416a8..cf36feb9 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( name="cwl-upgrader", - version="0.5", + version="1.0", description="Common Workflow Language standalone document upgrader", long_description=open(README).read(), author="Common Workflow Language contributors", @@ -27,8 +27,9 @@ package_data={"cwlupgrader.tests": ["*.cwl"]}, install_requires=["setuptools", "ruamel.yaml >= 0.14.12, < 0.16.8", "typing"], entry_points={"console_scripts": ["cwl-upgrader = cwlupgrader.main:main"]}, + python_requires=">=3.5, <4", classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Environment :: Console", "Intended Audience :: Developers", "Intended Audience :: Healthcare Industry", @@ -36,12 +37,11 @@ "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", "Topic :: Scientific/Engineering", "Topic :: Scientific/Engineering :: Bio-Informatics", ], diff --git a/tests/test_complete.py b/tests/test_complete.py index 9a8aaaf8..d20031dc 100644 --- a/tests/test_complete.py +++ b/tests/test_complete.py @@ -1,5 +1,7 @@ import filecmp + from cwlupgrader.main import main + from .util import get_data diff --git a/tests/util.py b/tests/util.py index 7a6184d2..b34119fd 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,8 +1,10 @@ -from __future__ import absolute_import import os -from pkg_resources import (Requirement, ResolutionError, # type: ignore - resource_filename) +from pkg_resources import ( # type: ignore + Requirement, + ResolutionError, + resource_filename, +) def get_data(filename): diff --git a/tox.ini b/tox.ini index acc6ab6f..68b0ab1d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,14 @@ [tox] envlist = - py{27,35,36,37,38}-lint - py{27,35,36,37,38}-unit - py{35,36,37,38}-mypy-{2,3} + py{35,36,37,38}-lint + py{35,36,37,38}-unit + py{35,36,37,38}-mypy skipsdist = True skip_missing_interpreters = True [travis] python = - 2.7: py27 3.5: py35 3.6: py36 3.7: py37 @@ -22,27 +21,25 @@ passenv = TRAVIS_* deps = -rrequirements.txt - py{27,35,36,37,38}-unit: pytest<5 - py{27,35,36,37,38}-unit: pytest-xdist - py{27}-lint: flake8 + py{35,36,37,38}-unit: pytest<5 + py{35,36,37,38}-unit: pytest-xdist py{35,36,37,38}-lint: flake8-bugbear py{36,37,38}-lint: black - mypy: mypy==0.600 + mypy: mypy==0.720 setenv = - py{27,35,36,37,38}-unit: LC_ALL = C + py{35,36,37,38}-unit: LC_ALL = C commands = - py{27,35,36,37,38}-unit: python -m pip install -U pip setuptools wheel - py{27,35,36,37,38}-unit: python -m pip install -e . + py{35,36,37,38}-unit: python -m pip install -U pip setuptools wheel + py{35,36,37,38}-unit: python -m pip install -e . unit: python setup.py test --addopts "--cov-report xml --cov cwlupgrader {posargs}" - py{27,35,36,37,38}-lint: flake8 cwlupgrader setup.py + py{35,36,37,38}-lint: flake8 cwlupgrader setup.py py{36,37,38}-lint: black --diff --check --target-version py27 cwlupgrader setup.py - py{35,36,37,38}-mypy2: make mypy2 - py{35,36,37,38}-mypy3: make mypy3 + py{35,36,37,38}-mypy: make mypy whitelist_externals = - py{27,35,36,37,38}-lint: flake8 - py{27,35,36,37,38}-lint: black - py{35,36,37,38}-mypy{2,3}: make + py{35,36,37,38}-lint: flake8 + py{35,36,37,38}-lint: black + py{35,36,37,38}-mypy: make lint: flake8