From d07162dce9bf717ff5a25215be76b48b92165262 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Mon, 10 Aug 2020 13:03:13 -0500 Subject: [PATCH 1/3] relax dependency version requirements to prepare for new pip resolver --- .pre-commit-config.yaml | 2 +- docker/Dockerfile | 2 +- docs/source/versions/v0_6_1.rst | 28 ++++++++++++++++++++++++++++ pyproject.toml | 22 ++++------------------ pytest.ini | 2 +- setup.cfg | 16 ++++++++-------- 6 files changed, 43 insertions(+), 29 deletions(-) create mode 100644 docs/source/versions/v0_6_1.rst diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b43cefaf..da9e0dbc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.1.0 + rev: v3.2.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer diff --git a/docker/Dockerfile b/docker/Dockerfile index ab4ea0da..b1fc2ce9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -67,6 +67,6 @@ ENTRYPOINT ["/home/mapper/htmap/docker/entrypoint.sh"] CMD ["bash"] COPY --chown=mapper:mapper . /home/${USER}/htmap -RUN python -m pip install --user --no-cache-dir --disable-pip-version-check "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.* +RUN python -m pip install --user --no-cache-dir --disable-pip-version-check --use-feature=2020-resolver "/home/${USER}/htmap[tests,docs]" htcondor==${HTCONDOR_VERSION}.* WORKDIR /home/${USER}/htmap diff --git a/docs/source/versions/v0_6_1.rst b/docs/source/versions/v0_6_1.rst new file mode 100644 index 00000000..61d698f1 --- /dev/null +++ b/docs/source/versions/v0_6_1.rst @@ -0,0 +1,28 @@ +v0.6.1 +====== + +.. py:currentmodule:: htmap + + +This version is a drop-in replacement for v0.6.0, except that it relaxes the +version requirements for several dependencies to accommodate upcoming changes +to the +`pip dependency resolver `_. + + +Known Issues +------------ + +* HTMap does not currently allow "directory content transfers", which is an HTCondor + feature where naming a directory in ``transfer_input_files`` with a trailing + slash transfers the contents of the directory, not the directory itself. + (If you try it, the directory itself will be transferred, as if you had not + used a trailing slash). + Issue: :issue:`215` +* Execution errors that result in the job being terminated but no output being + produced are still not handled entirely gracefully. Right now, the component + state will just show as ``ERRORED``, but there won't be an actual error report. +* Map component state may become corrupted when a map is manually vacated. + Force-removal may be needed to clean up maps if HTCondor and HTMap disagree + about the state of their components. + Issue: :issue:`129` diff --git a/pyproject.toml b/pyproject.toml index 832c0e36..eb8df6cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,22 +1,8 @@ +[build-system] +requires = ["setuptools >= 40.6.0", "wheel"] +build-backend = "setuptools.build_meta" + [tool.black] line-length = 100 target-version = ['py36', 'py37', 'py38'] include = '\.pyi?$' -exclude = ''' -/( - \.eggs - | \.git - | \.hg - | \.mypy_cache - | \.tox - | \.venv - | _build - | buck-out - | build - | dist -)/ -''' - -[build-system] -requires = ["setuptools >= 40.6.0", "wheel"] -build-backend = "setuptools.build_meta" diff --git a/pytest.ini b/pytest.ini index 0f81fb7f..feba99ce 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] -testspaths = tests +testpaths = tests console_output_style = count diff --git a/setup.cfg b/setup.cfg index 8e750a73..4e4d91b7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,17 +30,17 @@ python_requires = >=3.6 include_package_data = True install_requires = htcondor >= 8.8 - cloudpickle ~= 1.4 - toml ~= 0.10 - tqdm ~= 4.46 - click ~= 7.0 - click-didyoumean - halo - importlib-metadata ~= 1.0; python_version < "3.8" + cloudpickle >= 1.4 + toml >= 0.10 + tqdm >= 4.46 + click >= 7.0 + click-didyoumean>=0.0.3 + halo>=0.0.30 + importlib-metadata >= 1.0; python_version < "3.8" [options.extras_require] tests = - pytest + pytest>=6 pytest-mock pytest-xdist pytest-timeout From 6922e42fb28a2d8f4dc44215ce51390f141b92f1 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Mon, 10 Aug 2020 13:04:30 -0500 Subject: [PATCH 2/3] add a few new pre-commit checks --- .pre-commit-config.yaml | 12 ++++ htmap/__init__.py | 43 ++++-------- htmap/__main__.py | 1 - htmap/_startup.py | 4 +- htmap/checkpointing.py | 2 +- htmap/cli.py | 22 +++--- htmap/htio.py | 5 +- htmap/management.py | 15 ++-- htmap/mapped.py | 4 +- htmap/mapping.py | 22 ++---- htmap/maps.py | 42 +++--------- htmap/options.py | 13 ++-- htmap/run/_htmap_run.py | 10 +-- htmap/run/_htmap_transfer_plugin.py | 6 +- htmap/settings.py | 9 ++- htmap/state.py | 9 ++- htmap/tags.py | 8 +-- htmap/transfer.py | 10 ++- htmap/types.py | 3 +- htmap/utils.py | 25 ++----- htmap/version.py | 2 +- pyproject.toml | 6 ++ setup.cfg | 68 +++++++++---------- tests/cli/conftest.py | 1 - tests/conftest.py | 5 +- tests/integration/test_checkpointing.py | 2 +- tests/integration/test_errors_and_holds.py | 3 +- .../integration/test_late_materialization.py | 6 +- tests/integration/test_rerun.py | 4 +- .../test_automatic_input_file_transfer.py | 3 +- .../test_fixed_input_files.py | 4 +- .../user_file_transfer/test_output_files.py | 4 +- .../user_file_transfer/test_output_remaps.py | 7 +- .../test_url_input_transfer.py | 4 +- .../test_url_output_transfer.py | 6 +- tests/unit/test_base_descriptors.py | 4 +- .../test_delivery_method_base_descriptors.py | 4 +- tests/unit/test_delivery_method_setup.py | 4 +- tests/unit/test_htio.py | 3 +- tests/unit/test_options.py | 2 +- tests/unit/test_transfer_path.py | 4 +- tests/unit/test_transform_args_and_kwargs.py | 10 +-- tests/unit/test_wait_for_path_to_exist.py | 6 +- 43 files changed, 181 insertions(+), 246 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da9e0dbc..1e9a67b3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,3 +30,15 @@ repos: rev: v2.0.0 hooks: - id: hadolint + - repo: https://github.com/asottile/setup-cfg-fmt + rev: v1.11.0 + hooks: + - id: setup-cfg-fmt + - repo: https://github.com/asottile/seed-isort-config + rev: v2.2.0 + hooks: + - id: seed-isort-config + - repo: https://github.com/timothycrosley/isort + rev: 5.3.2 + hooks: + - id: isort diff --git a/htmap/__init__.py b/htmap/__init__.py index 63453214..643a3f03 100644 --- a/htmap/__init__.py +++ b/htmap/__init__.py @@ -16,47 +16,34 @@ import logging as _logging +from .settings import BASE_SETTINGS, USER_SETTINGS, settings from .version import __version__, version, version_info -from .settings import settings, USER_SETTINGS, BASE_SETTINGS - # SET UP NULL LOG HANDLER _logger = _logging.getLogger(__name__) _logger.setLevel(_logging.DEBUG) _logger.addHandler(_logging.NullHandler()) -from .mapping import ( - map, - starmap, - build_map, - MapBuilder, -) -from .mapped import mapped, MappedFunction -from .maps import ( - Map, - MapStdOut, - MapStdErr, - MapOutputFiles, -) -from .holds import ComponentHold +from . import _startup, exceptions +from .checkpointing import checkpoint from .errors import ComponentError -from .state import ComponentStatus -from .options import MapOptions, register_delivery_method +from .holds import ComponentHold from .management import ( - status, - status_json, - status_csv, + Transplant, + clean, load, load_maps, remove, - clean, - Transplant, - transplants, + status, + status_csv, + status_json, transplant_info, + transplants, ) +from .mapped import MappedFunction, mapped +from .mapping import MapBuilder, build_map, map, starmap +from .maps import Map, MapOutputFiles, MapStdErr, MapStdOut +from .options import MapOptions, register_delivery_method +from .state import ComponentStatus from .tags import get_tags -from .checkpointing import checkpoint from .transfer import TransferPath, transfer_output_files -from . import exceptions - -from . import _startup diff --git a/htmap/__main__.py b/htmap/__main__.py index f00bf09a..d7b8a34d 100644 --- a/htmap/__main__.py +++ b/htmap/__main__.py @@ -1,5 +1,4 @@ from .cli import cli - if __name__ == "__main__": exit(cli.main(prog_name="htmap")) diff --git a/htmap/_startup.py b/htmap/_startup.py index 18a64093..824a78f4 100644 --- a/htmap/_startup.py +++ b/htmap/_startup.py @@ -1,9 +1,9 @@ -import os import logging +import os from logging import handlers from pathlib import Path -from . import settings, names +from . import names, settings logger = logging.getLogger("htmap") diff --git a/htmap/checkpointing.py b/htmap/checkpointing.py index 3071d9d2..e9c50f8e 100644 --- a/htmap/checkpointing.py +++ b/htmap/checkpointing.py @@ -14,8 +14,8 @@ # limitations under the License. import os -from pathlib import Path import shutil +from pathlib import Path from . import names diff --git a/htmap/cli.py b/htmap/cli.py index b171cb5f..95212f16 100644 --- a/htmap/cli.py +++ b/htmap/cli.py @@ -13,28 +13,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, List, Collection, Tuple - -import logging -import sys -import time import collections -import random import functools +import logging +import random import shutil +import sys +import time from pathlib import Path - -import htcondor -import htmap -from htmap import names, __version__ -from htmap.management import _status, read_events +from typing import Collection, List, Optional, Tuple import click +import htcondor from click_didyoumean import DYMGroup - from halo import Halo from spinners import Spinners +import htmap +from htmap import __version__, names +from htmap.management import _status, read_events + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) diff --git a/htmap/htio.py b/htmap/htio.py index 8c1e1799..742eb215 100644 --- a/htmap/htio.py +++ b/htmap/htio.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, List, Iterator, Callable, Iterable -import logging - import gzip import json +import logging from pathlib import Path +from typing import Any, Callable, Iterable, Iterator, List import cloudpickle import htcondor diff --git a/htmap/management.py b/htmap/management.py index bb7c1778..e1ae1084 100644 --- a/htmap/management.py +++ b/htmap/management.py @@ -13,21 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Iterable, Dict, Union, NamedTuple, Callable, List, Optional -import logging - -from pathlib import Path -import datetime import collections -import json import csv +import datetime import io -import textwrap +import json +import logging import shutil +import textwrap import uuid from concurrent.futures.thread import ThreadPoolExecutor +from pathlib import Path +from typing import Callable, Dict, Iterable, List, NamedTuple, Optional, Tuple, Union -from . import maps, tags, mapping, utils, state, names, settings, exceptions +from . import exceptions, mapping, maps, names, settings, state, tags, utils logger = logging.getLogger(__name__) diff --git a/htmap/mapped.py b/htmap/mapped.py index fd13f2ad..1b1ba06f 100644 --- a/htmap/mapped.py +++ b/htmap/mapped.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Iterable, Dict, Union, Optional, Callable, Any import logging +from typing import Any, Callable, Dict, Iterable, Optional, Union -from . import mapping, options, maps +from . import mapping, maps, options logger = logging.getLogger(__name__) diff --git a/htmap/mapping.py b/htmap/mapping.py index cef94db0..3a64c104 100644 --- a/htmap/mapping.py +++ b/htmap/mapping.py @@ -13,30 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Iterable, Dict, Optional, Callable, Iterator, Any, List, Union +import itertools import logging - -import uuid import shutil +import uuid from pathlib import Path -import itertools from pprint import pformat +from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Tuple, Union import htcondor -from . import ( - htio, - tags, - exceptions, - maps, - transfer, - options, - condor, - settings, - names, - utils, -) -from .types import KWARGS, ARGS_OR_KWARGS, ARGS_AND_KWARGS, ARGS +from . import condor, exceptions, htio, maps, names, options, settings, tags, transfer, utils +from .types import ARGS, ARGS_AND_KWARGS, ARGS_OR_KWARGS, KWARGS logger = logging.getLogger(__name__) diff --git a/htmap/maps.py b/htmap/maps.py index d5128d46..6c3b723e 100644 --- a/htmap/maps.py +++ b/htmap/maps.py @@ -13,48 +13,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import ( - Tuple, - List, - Iterable, - Any, - Optional, - Iterator, - Dict, - Mapping, - MutableMapping, -) -import logging - +import collections +import collections.abc import datetime -import shutil -import time import functools import inspect -import collections -import collections.abc +import logging +import shutil +import time import weakref from copy import copy from pathlib import Path +from typing import Any, Dict, Iterable, Iterator, List, Mapping, MutableMapping, Optional, Tuple -from tqdm import tqdm - -import htcondor import classad +import htcondor +from tqdm import tqdm -from . import ( - htio, - state, - tags, - errors, - holds, - mapping, - condor, - settings, - utils, - names, - exceptions, -) +from . import condor, errors, exceptions, holds, htio, mapping, names, settings, state, tags, utils logger = logging.getLogger(__name__) diff --git a/htmap/options.py b/htmap/options.py index b268c5a3..cfe4f6ed 100644 --- a/htmap/options.py +++ b/htmap/options.py @@ -13,19 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Union, Iterable, Optional, Callable, Dict, List, Tuple -import logging - -import sys -import shutil import collections import hashlib +import logging +import shutil +import sys from pathlib import Path +from typing import Callable, Dict, Iterable, List, Optional, Tuple, Union import htcondor -from . import utils, transfer, exceptions, names, settings -from .types import TRANSFER_PATH, REMAPS +from . import exceptions, names, settings, transfer, utils +from .types import REMAPS, TRANSFER_PATH logger = logging.getLogger(__name__) diff --git a/htmap/run/_htmap_run.py b/htmap/run/_htmap_run.py index 8d9c8975..500b9b1e 100644 --- a/htmap/run/_htmap_run.py +++ b/htmap/run/_htmap_run.py @@ -15,16 +15,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime +import getpass +import gzip import os import shutil -import sys import socket -import datetime -import gzip +import subprocess +import sys import textwrap import traceback -import subprocess -import getpass from pathlib import Path TRANSFER_DIR = "_htmap_transfer" diff --git a/htmap/run/_htmap_transfer_plugin.py b/htmap/run/_htmap_transfer_plugin.py index f5872e44..8633f8ae 100755 --- a/htmap/run/_htmap_transfer_plugin.py +++ b/htmap/run/_htmap_transfer_plugin.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 +import contextlib +import pickle import subprocess import sys import textwrap -import contextlib -import pickle import traceback from pathlib import Path -import htcondor import classad +import htcondor TRANSFER_PLUGIN_CACHE = "_htmap_transfer_plugin_cache" USER_URL_TRANSFER_DIR = "_htmap_user_url_transfer" diff --git a/htmap/settings.py b/htmap/settings.py index aa4a49ec..96c9f79d 100644 --- a/htmap/settings.py +++ b/htmap/settings.py @@ -13,14 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Union, Any, Optional +import functools +import itertools import logging - import os -import itertools -import functools -from pathlib import Path from copy import copy +from pathlib import Path +from typing import Any, Optional, Union import toml diff --git a/htmap/state.py b/htmap/state.py index 42edadb2..fcbe69ca 100644 --- a/htmap/state.py +++ b/htmap/state.py @@ -13,17 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, Tuple, List -import logging - import datetime -import threading +import logging import pickle +import threading from pathlib import Path +from typing import Dict, List, Tuple import htcondor -from . import holds, names, utils, exceptions +from . import exceptions, holds, names, utils logger = logging.getLogger(__name__) diff --git a/htmap/tags.py b/htmap/tags.py index c6375abc..66405d92 100644 --- a/htmap/tags.py +++ b/htmap/tags.py @@ -13,15 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional - +import fnmatch import random import string from pathlib import Path -from typing import Tuple -import fnmatch +from typing import Optional, Tuple -from htmap import names, settings, exceptions +from htmap import exceptions, names, settings def tags_dir() -> Path: diff --git a/htmap/transfer.py b/htmap/transfer.py index b4377125..ccd2c674 100644 --- a/htmap/transfer.py +++ b/htmap/transfer.py @@ -13,17 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional, Union, Tuple - -import os -import shutil import functools +import os import pickle - +import shutil from pathlib import Path +from typing import Optional, Tuple, Union from urllib.parse import urlunsplit -from . import names, utils, exceptions +from . import exceptions, names, utils @functools.total_ordering diff --git a/htmap/types.py b/htmap/types.py index adc4a512..7cbb6213 100644 --- a/htmap/types.py +++ b/htmap/types.py @@ -1,6 +1,5 @@ -from typing import Dict, Any, Union, Tuple, Mapping - import os +from typing import Any, Dict, Mapping, Tuple, Union from htmap import transfer diff --git a/htmap/utils.py b/htmap/utils.py index 3fb6f280..ee66ce7e 100644 --- a/htmap/utils.py +++ b/htmap/utils.py @@ -13,33 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import ( - Optional, - Union, - Iterable, - Any, - Mapping, - MutableMapping, - Callable, - Dict, - Tuple, -) +import datetime +import enum import logging - import os -import time -import datetime +import re import subprocess import sys -import enum -import re +import time from pathlib import Path - -from . import exceptions +from typing import Any, Callable, Dict, Iterable, Mapping, MutableMapping, Optional, Tuple, Union import htcondor from classad import ClassAd +from . import exceptions + MutableMapping.register(ClassAd) logger = logging.getLogger(__name__) diff --git a/htmap/version.py b/htmap/version.py index 1fd2aeba..e19497da 100644 --- a/htmap/version.py +++ b/htmap/version.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Tuple, Optional +from typing import Optional, Tuple try: from importlib import metadata diff --git a/pyproject.toml b/pyproject.toml index eb8df6cc..b7ebb7dc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,3 +6,9 @@ build-backend = "setuptools.build_meta" line-length = 100 target-version = ['py36', 'py37', 'py38'] include = '\.pyi?$' + +[tool.isort] +known_third_party = ["classad", "click", "click_didyoumean", "cloudpickle", "halo", "htcondor", "pytest", "setuptools", "sphinx_rtd_theme", "spinners", "toml", "tqdm"] +line_length = 100 +multi_line_output = "VERTICAL_HANGING_INDENT" +include_trailing_comma = true diff --git a/setup.cfg b/setup.cfg index 4e4d91b7..95209ef6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,21 +1,21 @@ [metadata] name = htmap version = 0.6.0 -author = Josh Karpel -author_email = josh.karpel@gmail.com description = High-Throughput Computing in Python, powered by HTCondor long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/htcondor/htmap -license = ASL 2.0 +author = Josh Karpel +author_email = josh.karpel@gmail.com +license = Apache-2.0 license_file = LICENSE classifiers = - License :: OSI Approved :: Apache Software License Development Status :: 3 - Alpha - Programming Language :: Python :: 3 - Intended Audience :: Science/Research Intended Audience :: Developers + Intended Audience :: Science/Research + License :: OSI Approved :: Apache Software License Operating System :: OS Independent + Programming Language :: Python :: 3 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 @@ -26,45 +26,45 @@ classifiers = packages = htmap htmap.run -python_requires = >=3.6 -include_package_data = True install_requires = - htcondor >= 8.8 - cloudpickle >= 1.4 - toml >= 0.10 - tqdm >= 4.46 - click >= 7.0 + click>=7.0 click-didyoumean>=0.0.3 + cloudpickle>=1.4 halo>=0.0.30 - importlib-metadata >= 1.0; python_version < "3.8" + htcondor>=8.8 + toml>=0.10 + tqdm>=4.46 + importlib-metadata>=1.0;python_version < "3.8" +python_requires = >=3.6 +include_package_data = True + +[options.entry_points] +console_scripts = + htmap = htmap.cli:cli [options.extras_require] -tests = - pytest>=6 - pytest-mock - pytest-xdist - pytest-timeout - pytest-watch - pytest-profiling - coverage - pytest-cov - codecov - pre-commit docs = - sphinx - sphinx_rtd_theme - sphinx_autodoc_typehints - pygments-github-lexers ipython nbsphinx nbstripout + pygments-github-lexers + sphinx + sphinx-autobuild sphinx-click sphinx-issues - sphinx-autobuild - -[options.entry_points] -console_scripts = - htmap = htmap.cli:cli + sphinx_autodoc_typehints + sphinx_rtd_theme +tests = + codecov + coverage + pre-commit + pytest>=6 + pytest-cov + pytest-mock + pytest-profiling + pytest-timeout + pytest-watch + pytest-xdist [options.package_data] * = diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index b42cd186..b13151e9 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -16,7 +16,6 @@ import functools import pytest - from click.testing import CliRunner from htmap.cli import cli as htmap_cli diff --git a/tests/conftest.py b/tests/conftest.py index 790d9267..905fe307 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,15 +14,14 @@ # limitations under the License. import time -from pathlib import Path from copy import copy +from pathlib import Path import pytest import htmap -from htmap.settings import BASE_SETTINGS - from htmap._startup import ensure_htmap_dir_exists +from htmap.settings import BASE_SETTINGS # start with base settings (ignore user settings for tests) htmap.settings.replace(BASE_SETTINGS) diff --git a/tests/integration/test_checkpointing.py b/tests/integration/test_checkpointing.py index 2386b746..d9d84627 100644 --- a/tests/integration/test_checkpointing.py +++ b/tests/integration/test_checkpointing.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path import time +from pathlib import Path import pytest diff --git a/tests/integration/test_errors_and_holds.py b/tests/integration/test_errors_and_holds.py index d846deca..2ecb8832 100644 --- a/tests/integration/test_errors_and_holds.py +++ b/tests/integration/test_errors_and_holds.py @@ -15,12 +15,11 @@ from pathlib import Path +import htcondor import pytest import htmap -import htcondor - @pytest.fixture(scope="function") def hold_before_error(): diff --git a/tests/integration/test_late_materialization.py b/tests/integration/test_late_materialization.py index 1bfd23e7..d8b402ee 100644 --- a/tests/integration/test_late_materialization.py +++ b/tests/integration/test_late_materialization.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import time - from pathlib import Path -import htmap import htcondor +import pytest + +import htmap TIMEOUT = 300 diff --git a/tests/integration/test_rerun.py b/tests/integration/test_rerun.py index 0925540c..84c27946 100644 --- a/tests/integration/test_rerun.py +++ b/tests/integration/test_rerun.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - import time from pathlib import Path +import pytest + import htmap TIMEOUT = 300 diff --git a/tests/integration/user_file_transfer/test_automatic_input_file_transfer.py b/tests/integration/user_file_transfer/test_automatic_input_file_transfer.py index 7eb9dae0..c9af7eee 100644 --- a/tests/integration/user_file_transfer/test_automatic_input_file_transfer.py +++ b/tests/integration/user_file_transfer/test_automatic_input_file_transfer.py @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path from typing import Set import pytest -from pathlib import Path - import htmap TEST_FILE_NAME = "test_file" diff --git a/tests/integration/user_file_transfer/test_fixed_input_files.py b/tests/integration/user_file_transfer/test_fixed_input_files.py index 5beed98b..452bfd12 100644 --- a/tests/integration/user_file_transfer/test_fixed_input_files.py +++ b/tests/integration/user_file_transfer/test_fixed_input_files.py @@ -14,10 +14,10 @@ # limitations under the License. -import pytest - from pathlib import Path +import pytest + import htmap TIMEOUT = 300 diff --git a/tests/integration/user_file_transfer/test_output_files.py b/tests/integration/user_file_transfer/test_output_files.py index 2c609835..c9b6e4c4 100644 --- a/tests/integration/user_file_transfer/test_output_files.py +++ b/tests/integration/user_file_transfer/test_output_files.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from pathlib import Path +import pytest + import htmap TIMEOUT = 300 diff --git a/tests/integration/user_file_transfer/test_output_remaps.py b/tests/integration/user_file_transfer/test_output_remaps.py index 43a4a741..c9bcc5bc 100644 --- a/tests/integration/user_file_transfer/test_output_remaps.py +++ b/tests/integration/user_file_transfer/test_output_remaps.py @@ -14,15 +14,14 @@ # limitations under the License. -import pytest - from pathlib import Path +import htcondor +import pytest + import htmap from htmap import utils -import htcondor - TIMEOUT = 300 diff --git a/tests/integration/user_file_transfer/test_url_input_transfer.py b/tests/integration/user_file_transfer/test_url_input_transfer.py index 27bc97bf..c7c839b2 100644 --- a/tests/integration/user_file_transfer/test_url_input_transfer.py +++ b/tests/integration/user_file_transfer/test_url_input_transfer.py @@ -14,10 +14,10 @@ # limitations under the License. -import pytest - from pathlib import Path +import pytest + import htmap TIMEOUT = 300 diff --git a/tests/integration/user_file_transfer/test_url_output_transfer.py b/tests/integration/user_file_transfer/test_url_output_transfer.py index b0bbbf07..46df62c1 100644 --- a/tests/integration/user_file_transfer/test_url_output_transfer.py +++ b/tests/integration/user_file_transfer/test_url_output_transfer.py @@ -14,13 +14,13 @@ # limitations under the License. -import pytest - from pathlib import Path +import htcondor +import pytest + import htmap from htmap import utils -import htcondor TIMEOUT = 300 diff --git a/tests/unit/test_base_descriptors.py b/tests/unit/test_base_descriptors.py index 0bc526c3..c591f337 100644 --- a/tests/unit/test_base_descriptors.py +++ b/tests/unit/test_base_descriptors.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from pathlib import Path +import pytest + import htmap from htmap.options import ( get_base_descriptors, diff --git a/tests/unit/test_delivery_method_base_descriptors.py b/tests/unit/test_delivery_method_base_descriptors.py index 98560f8f..afce5b32 100644 --- a/tests/unit/test_delivery_method_base_descriptors.py +++ b/tests/unit/test_delivery_method_base_descriptors.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from pathlib import Path +import pytest + import htmap from htmap.options import get_base_descriptors diff --git a/tests/unit/test_delivery_method_setup.py b/tests/unit/test_delivery_method_setup.py index acfd6694..51590eee 100644 --- a/tests/unit/test_delivery_method_setup.py +++ b/tests/unit/test_delivery_method_setup.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from pathlib import Path +import pytest + import htmap from htmap.options import run_delivery_setup diff --git a/tests/unit/test_htio.py b/tests/unit/test_htio.py index fc17b4fd..7cc78456 100644 --- a/tests/unit/test_htio.py +++ b/tests/unit/test_htio.py @@ -13,11 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from pathlib import Path import htcondor +import pytest from htmap import htio diff --git a/tests/unit/test_options.py b/tests/unit/test_options.py index f0d624a1..ef4863a6 100644 --- a/tests/unit/test_options.py +++ b/tests/unit/test_options.py @@ -16,10 +16,10 @@ from pathlib import Path import pytest -from tests.conftest import exception_msg import htmap from htmap.options import create_submit_object_and_itemdata, get_base_descriptors +from tests.conftest import exception_msg def test_one_reserved_kwarg_raises(): diff --git a/tests/unit/test_transfer_path.py b/tests/unit/test_transfer_path.py index 698d3698..4610af57 100644 --- a/tests/unit/test_transfer_path.py +++ b/tests/unit/test_transfer_path.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from pathlib import Path +import pytest + from htmap import TransferPath diff --git a/tests/unit/test_transform_args_and_kwargs.py b/tests/unit/test_transform_args_and_kwargs.py index af87e98a..7203fd4e 100644 --- a/tests/unit/test_transform_args_and_kwargs.py +++ b/tests/unit/test_transform_args_and_kwargs.py @@ -13,16 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from pathlib import Path +import pytest + from htmap import TransferPath -from htmap.mapping import ( - transform_args_and_kwargs, - transform_input_paths, - transform_input_path, -) +from htmap.mapping import transform_args_and_kwargs, transform_input_path, transform_input_paths @pytest.mark.parametrize( diff --git a/tests/unit/test_wait_for_path_to_exist.py b/tests/unit/test_wait_for_path_to_exist.py index 7a92e22b..f7723446 100644 --- a/tests/unit/test_wait_for_path_to_exist.py +++ b/tests/unit/test_wait_for_path_to_exist.py @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - from datetime import timedelta from pathlib import Path +import pytest + import htmap -from htmap.utils import wait_for_path_to_exist, timeout_to_seconds +from htmap.utils import timeout_to_seconds, wait_for_path_to_exist def test_returns_when_path_does_exist(): From fa6bddb565ed044592b301d56c8b342af86cde05 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Mon, 10 Aug 2020 13:04:42 -0500 Subject: [PATCH 3/3] bump version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 95209ef6..c7efa4db 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = htmap -version = 0.6.0 +version = 0.6.1 description = High-Throughput Computing in Python, powered by HTCondor long_description = file: README.md long_description_content_type = text/markdown