diff --git a/.flake8 b/.flake8 index 09c47a3c..d6074647 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,19 @@ [flake8] -max-line-length = 80 +ignore= + # Whitespace before ':' + E203 + # Too many leading '#' for block comment + E266 + # Line break occurred before a binary operator + W503 + # unindexed parameters in the str.format, see: + # https://pypi.org/project/flake8-string-format/ + P1 +max_line_length = 88 +max-complexity = 15 +select = B,C,E,F,W,T4,B902,T,P +show_source = true +count = true per-file-ignores = upath/__init__.py: F401 exclude = @@ -10,5 +24,3 @@ exclude = .github, .gitignore, .pytest_cache, - - \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6313b56c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..247ad054 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,28 @@ +version: 2 + +updates: + - directory: "/" + package-ecosystem: "pip" + schedule: + interval: "weekly" + labels: + - "maintenance" + # Update via cruft + ignore: + - dependency-name: "mkdocs*" + - dependency-name: "pytest*" + - dependency-name: "pylint" + - dependency-name: "mypy" + + - directory: "/" + package-ecosystem: "github-actions" + schedule: + interval: "weekly" + labels: + - "maintenance" + # Update via cruft + ignore: + - dependency-name: "actions/checkout" + - dependency-name: "actions/setup-python" + - dependency-name: "pypa/gh-action-pypi-publish" + - dependency-name: "codecov/codecov-action" diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index 170968e9..00000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Python package -on: - push: - branches: - - main - tags: - - v*.*.* - pull_request: -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] - os: [ubuntu-latest, windows-latest] - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip nox - nox -s install - - name: Test with pytest - run: | - nox -s smoke - - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - - name: Install dependencies - run: | - python -m pip install --upgrade pip nox - - name: Lint - run: | - nox -s lint - - name: Type Checking - run: | - nox -s type_checking - - build: - needs: [tests, lint] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - - name: Install dependencies - run: | - python -m pip install --upgrade pip nox - - name: build package - run: | - nox -s build - - name: Publish package - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') && runner.os == 'Linux' - uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: __token__ - password: ${{ secrets.UPATH_GIT_REPO }} - verbose: true - skip_existing: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..5152f753 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,41 @@ +name: Release + +on: + release: + types: [published] + workflow_dispatch: + +env: + FORCE_COLOR: "1" + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Upgrade pip and nox + run: | + pip install --upgrade pip nox + pip --version + nox --version + + - name: Build package + run: nox -s build + + - name: Upload package + if: github.event_name == 'release' + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.UPATH_GIT_REPO }} + verbose: true + skip_existing: true diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 00000000..41e6eba9 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,90 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +env: + FORCE_COLOR: "1" + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + tests: + timeout-minutes: 10 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-20.04, windows-latest, macos-latest] + pyv: ['3.8', '3.9', '3.10', '3.11'] + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.pyv }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.pyv }} + + - name: Upgrade pip and nox + run: | + python -m pip install --upgrade pip nox + pip --version + nox --version + + - name: Run tests + run: nox -s tests-${{ matrix.nox_pyv || matrix.pyv }} -- --cov-report=xml + + lint: + runs-on: ubuntu-latest + + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.pyv }} + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Upgrade pip and nox + run: | + python -m pip install --upgrade pip nox + pip --version + nox --version + + - name: Lint code and check dependencies + run: nox -s lint safety + + build: + needs: [tests, lint] + runs-on: ubuntu-latest + steps: + - name: Check out the repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.pyv }} + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip nox + pip --version + nox --version + + - name: Build package + run: nox -s build diff --git a/.gitignore b/.gitignore index 75389db1..e3312579 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,141 @@ -**/__pycache__ -**/dist -**/build -**.pyc \ No newline at end of file +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# setuptools_scm +upath/_version.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..45cbdd24 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,54 @@ +default_language_version: + python: python3 +repos: + - repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + args: ['--assume-in-merge'] + - id: check-toml + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + args: ['--fix=lf'] + - id: sort-simple-yaml + - id: trailing-whitespace + - repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell + additional_dependencies: ["tomli"] + - repo: https://github.com/asottile/pyupgrade + rev: v3.6.0 + hooks: + - id: pyupgrade + args: [--py38-plus] + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + additional_dependencies: + - flake8-bugbear==23.1.20 + - flake8-comprehensions==3.10.1 + - flake8-debugger==4.1.2 + - flake8-string-format==0.3.0 + - repo: https://github.com/pycqa/bandit + rev: 1.7.5 + hooks: + - id: bandit + args: [-c, pyproject.toml] + additional_dependencies: ["tomli>=1.1.0"] diff --git a/CODE_OF_CONDUCT.rst b/CODE_OF_CONDUCT.rst new file mode 100644 index 00000000..859325c0 --- /dev/null +++ b/CODE_OF_CONDUCT.rst @@ -0,0 +1,105 @@ +Contributor Covenant Code of Conduct +==================================== + +Our Pledge +---------- + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + + +Our Standards +------------- + +Examples of behavior that contributes to a positive environment for our community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or + advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email + address, without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +Enforcement Responsibilities +---------------------------- + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. + + +Scope +----- + +This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. + + +Enforcement +----------- + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at andrewfulton9gmail.com. All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the reporter of any incident. + + +Enforcement Guidelines +---------------------- + +Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: + + +1. Correction +~~~~~~~~~~~~~ + +**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. + + +2. Warning +~~~~~~~~~~ + +**Community Impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. + + +3. Temporary Ban +~~~~~~~~~~~~~~~~ + +**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. + + +4. Permanent Ban +~~~~~~~~~~~~~~~~ + +**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the community. + + +Attribution +----------- + +This Code of Conduct is adapted from the `Contributor Covenant `__, version 2.0, +available at https://www.contributor-covenant.org/version/2/0/code_of_conduct/. + +Community Impact Guidelines were inspired by `Mozilla’s code of conduct enforcement ladder `__. + +.. _homepage: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see the FAQ at +https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 00000000..58063d3d --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,109 @@ +Contributor Guide +================= + +Thank you for your interest in improving this project. +This project is open-source under the `MIT license`_ and +welcomes contributions in the form of bug reports, feature requests, and pull requests. + +Here is a list of important resources for contributors: + +- `Source Code`_ +- `Issue Tracker`_ +- `Code of Conduct`_ + +.. _MIT license: https://opensource.org/licenses/MIT +.. _Source Code: https://github.com/fsspec/universal_pathlib +.. _Issue Tracker: https://github.com/fsspec/universal_pathlib/issues + +How to report a bug +------------------- + +Report bugs on the `Issue Tracker`_. + +When filing an issue, make sure to answer these questions: + +- Which operating system and Python version are you using? +- Which version of this project are you using? +- What did you do? +- What did you expect to see? +- What did you see instead? + +The best way to get your bug fixed is to provide a test case, +and/or steps to reproduce the issue. + + +How to request a feature +------------------------ + +Request features on the `Issue Tracker`_. + + +How to set up your development environment +------------------------------------------ + +You need Python 3.8+ and the following tools: + +- Nox_ + +Install the package with development requirements: + +.. code:: console + + $ pip install nox + +.. _Nox: https://nox.thea.codes/ + + +How to test the project +----------------------- + +Run the full test suite: + +.. code:: console + + $ nox + +List the available Nox sessions: + +.. code:: console + + $ nox --list-sessions + +You can also run a specific Nox session. +For example, invoke the unit test suite like this: + +.. code:: console + + $ nox --session=tests + +Unit tests are located in the ``tests`` directory, +and are written using the pytest_ testing framework. + +.. _pytest: https://pytest.readthedocs.io/ + + +How to submit changes +--------------------- + +Open a `pull request`_ to submit changes to this project. + +Your pull request needs to meet the following guidelines for acceptance: + +- The Nox test suite must pass without errors and warnings. +- Include unit tests. This project maintains 100% code coverage. +- If your changes add functionality, update the documentation accordingly. + +Feel free to submit early, though—we can always iterate on this. + +To run linting and code formatting checks, you can invoke a `lint` session in nox: + +.. code:: console + + $ nox -s lint + +It is recommended to open an issue before starting work on anything. +This will allow a chance to talk it over with the owners and validate your approach. + +.. _pull request: https://github.com/fsspec/universal_pathlib/pulls +.. github-only +.. _Code of Conduct: CODE_OF_CONDUCT.rst diff --git a/LICENSE b/LICENSE index 91bad1de..b32d8095 100644 --- a/LICENSE +++ b/LICENSE @@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..602f79e7 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +exclude .git* +recursive-exclude .git * +recursive-exclude .github * diff --git a/README.md b/README.md index 9b25b739..b1837918 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,14 @@ # Universal Pathlib +[![PyPI](https://img.shields.io/pypi/v/universal_pathlib.svg)](https://pypi.org/project/universal_pathlib/) +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/universal_pathlib)](https://pypi.org/project/universal_pathlib/) +[![PyPI - License](https://img.shields.io/pypi/l/universal_pathlib)](https://github.com/fsspec/universal_pathlib/blob/main/LICENSE) +[![Conda (channel only)](https://img.shields.io/conda/vn/conda-forge/universal_pathlib?label=conda)](https://anaconda.org/conda-forge/universal_pathlib) + +[![Tests](https://github.com/fsspec/universal_pathlib/actions/workflows/python.yml/badge.svg)](https://github.com/fsspec/universal_pathlib/workflows/Tests/badge.svg) +[![GitHub issues](https://img.shields.io/github/issues/fsspec/universal_pathlib)](https://github.com/fsspec/universal_pathlib/issues) +[![Codestyle black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + Universal Pathlib is a python library that aims to extend Python's built-in [`pathlib.Path`](https://docs.python.org/3/library/pathlib.html) api to use a variety of backend filesystems using [`fsspec`](https://filesystem-spec.readthedocs.io/en/latest/intro.html) ## Installation @@ -51,6 +60,17 @@ For more examples, see the [example notebook here](notebooks/examples.ipynb) Other fsspec-compatible filesystems may also work, but are not supported and tested. Contributions for new filesystems are welcome! +## Contributing + +Contributions are very welcome. +To learn more, see the [Contributor Guide](CONTRIBUTING.rst). + ## License -MIT License +Distributed under the terms of the [MIT license](LICENSE), +*universal_pathlib* is free and open source software. + +## Issues + +If you encounter any problems, +please [file an issue](https://github.com/fsspec/universal_pathlib/issues) along with a detailed description. diff --git a/environment.yml b/environment.yml index 0e984d67..3ae4d15a 100644 --- a/environment.yml +++ b/environment.yml @@ -20,4 +20,3 @@ dependencies: - hadoop-test-cluster - gcsfs - nox - \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index dd070a59..1ef5b9d5 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,74 +1,82 @@ -import sys +"""Automation using nox.""" +import glob +import os import nox -from pathlib import Path +nox.options.reuse_existing_virtualenvs = True +nox.options.sessions = "lint", "tests" +locations = ("upath",) -@nox.session() -def develop(session): - session.install("flit") - session.run(*"flit install -s".split()) +@nox.session(python=["3.8", "3.9", "3.10", "3.11", "pypy3.8", "pypy3.9"]) +def tests(session: nox.Session) -> None: + session.install(".[dev]") + session.run( + "pytest", + "-m", + "not hdfs", + "--cov", + "--cov-config=pyproject.toml", + *session.posargs, + env={"COVERAGE_FILE": f".coverage.{session.python}"}, + ) -@nox.session() -def black(session): - session.install("black") - session.run(*"black upath noxfile.py setup.py".split()) +@nox.session +def lint(session: nox.Session) -> None: + session.install("pre-commit") + session.install("-e", ".[tests]") -@nox.session() -def lint(session): - session.install("flake8") - session.run("flake8", "upath") + args = *(session.posargs or ("--show-diff-on-failure",)), "--all-files" + session.run("pre-commit", "run", *args) + session.run("python", "-m", "mypy") + # session.run("python", "-m", "pylint", *locations) -@nox.session() -def type_checking(session): - session.install("mypy") - session.run("mypy", "upath") +@nox.session +def safety(session: nox.Session) -> None: + """Scan dependencies for insecure packages.""" + session.install(".") + session.install("safety") + session.run("safety", "check", "--full-report") -@nox.session() -def install(session): - session.install(".") +@nox.session +def build(session: nox.Session) -> None: + session.install("build", "setuptools", "twine") + session.run("python", "-m", "build") + dists = glob.glob("dist/*") + session.run("twine", "check", *dists, silent=True) -@nox.session() -def smoke(session): - if (3, 10) < sys.version_info <= (3, 11, 0, "final"): - # workaround for missing aiohttp wheels for py3.11 - session.install( - "aiohttp", - "--no-binary", - "aiohttp", - env={"AIOHTTP_NO_EXTENSIONS": "1"}, - ) - - session.install( - "pytest", - "adlfs", - "aiohttp", - "requests", - "gcsfs", - "s3fs", - "moto[s3,server]", - "webdav4[fsspec]", - "wsgidav", - "cheroot", - ) - session.run(*"pytest --skiphdfs -vv upath".split()) +@nox.session +def develop(session: nox.Session) -> None: + """Sets up a python development environment for the project.""" + args = session.posargs or ("venv",) + venv_dir = os.fsdecode(os.path.abspath(args[0])) + session.log(f"Setting up virtual environment in {venv_dir}") + session.install("virtualenv") + session.run("virtualenv", venv_dir, silent=True) + + python = os.path.join(venv_dir, "bin/python") + session.run(python, "-m", "pip", "install", "-e", ".[dev]", external=True) -@nox.session() -def build(session): - session.install("flit") - session.run(*"flit build".split()) + +@nox.session +def black(session): + print("please run `nox -s lint` instead") + raise SystemExit(1) + + +@nox.session +def type_checking(session): + print("please run `nox -s lint` instead") + raise SystemExit(1) @nox.session() -def rm_dirs(session): - paths = ["build", "dist"] - for path in paths: - p = Path(path) - if p.exists(): - session.run(*f"rm -rf {str(p)}".split()) +def smoke(session): + print("please tun `nox -s tests` instead") + raise SystemExit(1) diff --git a/pyproject.toml b/pyproject.toml index ae51bbe4..d834d1ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,61 +1,67 @@ [build-system] -build-backend = "flit_core.buildapi" -requires = ["flit_core>=2,<4"] +requires = ["setuptools>=48", "setuptools_scm[toml]>=6.3.1"] +build-backend = "setuptools.build_meta" -[tool] -[tool.flit] -[tool.flit.metadata] -dist-name = "universal_pathlib" -author = "Andrew Fulton" -author-email = "andrewfulton9@gmail.com" -classifiers = [] -home-page = "https://github.com/fsspec/universal_pathlib" -keywords = "" -license = "MIT" -maintainer = "Andrew Fulton" -maintainer-email = "andrewfulton9@gmail.com" -module = "upath" -requires = ["fsspec"] -requires-python = ">=3.7" -description-file = "README.md" +[tool.setuptools_scm] +write_to = "upath/_version.py" +version_scheme = "post-release" -[tool.flit.metadata.urls] - -[tool.flit.metadata.requires-extra] -dev = [] -doc = [] -test = [ - "aiohttp", - "adlfs", - "flake8", - "gcsfs", - "hadoop-test-cluster", - "ipython", - "jupyter", - "moto", - "pyarrow", - "pylint", - "pytest", - "requests", - "s3fs", - "webdav4[fsspec]" -] +[tool.black] +line-length = 88 +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist +)/ +''' -[tool.flit.scripts] +[tool.isort] +profile = "black" +known_first_party = ["upath"] +force_single_line = true +line_length = 88 -[tool.flit.sdist] -include = [] +[tool.pytest.ini_options] +addopts = "-ra" -[tool.flit.entrypoints] +[tool.coverage.run] +branch = true +source = ["upath"] -[tool.black] -line-length = 80 -target-version = ['py38'] +[tool.coverage.report] +show_missing = true +exclude_lines = [ + "pragma: no cover", + "if __name__ == .__main__.:", + "if typing.TYPE_CHECKING:", + "if TYPE_CHECKING:", + "raise NotImplementedError", + "raise AssertionError", + "@overload", +] [tool.mypy] -python_version = "3.7" -warn_return_any = true -warn_unused_configs = true +# Error output +show_column_numbers = true +show_error_codes = true +show_error_context = true +show_traceback = true +pretty = true +check_untyped_defs = false +# Warnings +warn_no_return = true +warn_redundant_casts = true +warn_unreachable = true +files = ["upath"] exclude = "^notebooks|^venv.*|tests.*|^noxfile.py" [[tool.mypy.overrides]] @@ -66,6 +72,19 @@ ignore_missing_imports = true module = "webdav4.fsspec.*" ignore_missing_imports = true -[[tool.mypy.overrides]] -module = "setuptools.*" -ignore_missing_imports = true +[tool.pylint.format] +max-line-length = 88 + +[tool.pylint.message_control] +enable = ["c-extension-no-member", "no-else-return"] + +[tool.pylint.variables] +dummy-variables-rgx = "_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_" +ignored-argument-names = "_.*|^ignored_|^unused_|args|kwargs" + +[tool.codespell] +ignore-words-list = " " + +[tool.bandit] +exclude_dirs = ["tests"] +skips = ["B101"] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..0f3f9b4e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,57 @@ +[metadata] +description = pathlib api extended to use fsspec backends +name = universal_pathlib +long_description = file: README.md +long_description_content_type = text/markdown +license = MIT +license_files = + LICENSE +url = https://github.com/fsspec/universal_pathlib +platforms=any +authors = Andrew Fulton +maintainer_email = andrewfulton9gmail.com +classifiers = + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Development Status :: 4 - Beta + +[options] +python_requires = >=3.8 +zip_safe = False +packages = find: +install_requires= + fsspec + +[options.extras_require] +tests = + pytest==7.3.2 + pytest-sugar==0.9.6 + pytest-cov==4.1.0 + pytest-mock==3.11.1 + pylint==2.17.4 + mypy==1.3.0 +dev = + %(tests)s + adlfs + aiohttp + requests + gcsfs + s3fs + moto[s3,server] + webdav4[fsspec] + wsgidav + cheroot + hadoop-test-cluster + pyarrow + +[options.package_data] +upath = + py.typed + +[options.packages.find] +exclude = + upath.tests + upath.tests.* diff --git a/setup.py b/setup.py deleted file mode 100644 index 485fed85..00000000 --- a/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -import setuptools - -from upath import __version__ - -with open("README.md") as f: - long_description = f.read() - -setuptools.setup( - name="universal_pathlib", - version=__version__, - author="Andrew Fulton", - author_email="andrewfulton9@gmail.com", - url="https://github.com/fsspec/universal_pathlib", - packages=setuptools.find_packages(), - python_requires=">=3.7", - description="pathlib api extended to use fsspec backends", - long_description=long_description, - long_description_content_type="text/markdown", - license="MIT", -) diff --git a/upath/__init__.py b/upath/__init__.py index 59d1006f..a9bccf47 100644 --- a/upath/__init__.py +++ b/upath/__init__.py @@ -1,7 +1,9 @@ """Pathlib API extended to use fsspec backends.""" -__version__ = "0.0.23" - from upath.core import UPath +try: + from upath._version import __version__ +except ImportError: + __version__ = "not-installed" __all__ = ["UPath"] diff --git a/upath/core.py b/upath/core.py index b4023b33..0481cc9f 100644 --- a/upath/core.py +++ b/upath/core.py @@ -3,12 +3,12 @@ import re import sys from os import PathLike -from pathlib import _PosixFlavour # type: ignore from pathlib import Path from pathlib import PurePath +from pathlib import _PosixFlavour # type: ignore +from typing import TYPE_CHECKING from typing import Sequence from typing import TypeVar -from typing import TYPE_CHECKING from urllib.parse import urlsplit from urllib.parse import urlunsplit @@ -74,9 +74,7 @@ def info(self, path, **kwargs): return self._fs.info(self._format_path(path), **kwargs) def rm(self, path, recursive, **kwargs): - return self._fs.rm( - self._format_path(path), recursive=recursive, **kwargs - ) + return self._fs.rm(self._format_path(path), recursive=recursive, **kwargs) def mkdir(self, path, create_parents=True, **kwargs): return self._fs.mkdir( @@ -84,9 +82,7 @@ def mkdir(self, path, create_parents=True, **kwargs): ) def makedirs(self, path, exist_ok=False, **kwargs): - return self._fs.makedirs( - self._format_path(path), exist_ok=exist_ok, **kwargs - ) + return self._fs.makedirs(self._format_path(path), exist_ok=exist_ok, **kwargs) def touch(self, path, **kwargs): return self._fs.touch(self._format_path(path), **kwargs) @@ -199,9 +195,7 @@ def __getattr__(self, item: str) -> Any: if item == "_accessor": # cache the _accessor attribute on first access kwargs = self._kwargs.copy() - self._accessor = _accessor = self._default_accessor( - self._url, **kwargs - ) + self._accessor = _accessor = self._default_accessor(self._url, **kwargs) return _accessor else: raise AttributeError(item) @@ -211,9 +205,7 @@ def _make_child(self: PT, args: list[str]) -> PT: drv, root, parts = self._flavour.join_parsed_parts( self._drv, self._root, self._parts, drv, root, parts ) - return self._from_parsed_parts( - drv, root, parts, url=self._url, **self._kwargs - ) + return self._from_parsed_parts(drv, root, parts, url=self._url, **self._kwargs) def _make_child_relpath(self: PT, part: str) -> PT: # This is an optimization used for dir walking. `part` must be @@ -253,9 +245,7 @@ def _format_parsed_parts( @property def path(self) -> str: if self._parts: - join_parts = ( - self._parts[1:] if self._parts[0] == "/" else self._parts - ) + join_parts = self._parts[1:] if self._parts[0] == "/" else self._parts path: str = self._flavour.join(join_parts) return self._root + path else: @@ -389,14 +379,15 @@ def resolve(self: PT, strict: bool = False) -> PT: def exists(self) -> bool: """Check whether this path exists or not.""" - if not getattr(self._accessor, "exists"): + accessor = self._accessor + try: + return bool(accessor.exists(self)) + except AttributeError: try: self._accessor.stat(self) except FileNotFoundError: return False return True - else: - return bool(self._accessor.exists(self)) def is_dir(self) -> bool: try: @@ -663,12 +654,12 @@ def with_suffix(self: PT, suffix: str) -> PT: """ f = self._flavour if f.sep in suffix or f.altsep and f.altsep in suffix: - raise ValueError("Invalid suffix %r" % (suffix,)) + raise ValueError(f"Invalid suffix {suffix!r}") if suffix and not suffix.startswith(".") or suffix == ".": raise ValueError("Invalid suffix %r" % (suffix)) name = self.name if not name: - raise ValueError("%r has an empty name" % (self,)) + raise ValueError(f"{self!r} has an empty name") old_suffix = self.suffix if not old_suffix: name = name + suffix @@ -685,7 +676,7 @@ def with_suffix(self: PT, suffix: str) -> PT: def with_name(self: PT, name: str) -> PT: """Return a new path with the file name changed.""" if not self.name: - raise ValueError("%r has an empty name" % (self,)) + raise ValueError(f"{self!r} has an empty name") drv, root, parts = self._flavour.parse_parts((name,)) if ( not name @@ -754,4 +745,4 @@ def __getitem__(self, idx): ) def __repr__(self): - return "<{}.parents>".format(self._pathcls.__name__) + return f"<{self._pathcls.__name__}.parents>" diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index d2f12bf7..b3fffd6d 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import re import upath.core diff --git a/upath/implementations/hdfs.py b/upath/implementations/hdfs.py index 5a28573e..c00f8931 100644 --- a/upath/implementations/hdfs.py +++ b/upath/implementations/hdfs.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import upath.core diff --git a/upath/implementations/http.py b/upath/implementations/http.py index ce9bcbbe..14c14b3f 100644 --- a/upath/implementations/http.py +++ b/upath/implementations/http.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from fsspec.asyn import sync import upath.core @@ -51,8 +53,8 @@ def _sub_path(self, name): return name def resolve( - self: "HTTPPath", strict: bool = False, follow_redirects: bool = True - ) -> "HTTPPath": + self: HTTPPath, strict: bool = False, follow_redirects: bool = True + ) -> HTTPPath: """Normalize the path and resolve redirects.""" # Normalise the path resolved_path = super().resolve(strict=strict) diff --git a/upath/implementations/memory.py b/upath/implementations/memory.py index 994cd9bb..4d7d8bd0 100644 --- a/upath/implementations/memory.py +++ b/upath/implementations/memory.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import upath.core diff --git a/upath/implementations/webdav.py b/upath/implementations/webdav.py index ac479e93..d665dce1 100644 --- a/upath/implementations/webdav.py +++ b/upath/implementations/webdav.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from urllib.parse import ParseResult from urllib.parse import urlunsplit diff --git a/upath/registry.py b/upath/registry.py index 206db3e8..ed4567a4 100644 --- a/upath/registry.py +++ b/upath/registry.py @@ -8,7 +8,6 @@ from fsspec.core import get_filesystem_class - if TYPE_CHECKING: from upath.core import PT @@ -47,7 +46,7 @@ def __getitem__(self, item: str) -> type[PT] | None: _registry = _Registry() -@lru_cache() +@lru_cache def get_upath_class(protocol: str) -> type[PT] | type[Path] | None: """Return the upath cls for the given protocol.""" cls: type[PT] | None = _registry[protocol] @@ -70,4 +69,4 @@ def get_upath_class(protocol: str) -> type[PT] | type[Path] | None: stacklevel=2, ) mod = importlib.import_module("upath.core") - return getattr(mod, "UPath") # type: ignore + return mod.UPath # type: ignore diff --git a/upath/tests/cases.py b/upath/tests/cases.py index 9f00d8f5..ba0fbcb8 100644 --- a/upath/tests/cases.py +++ b/upath/tests/cases.py @@ -1,9 +1,10 @@ import pickle -from pathlib import Path import re import sys +from pathlib import Path import pytest + from upath import UPath @@ -44,13 +45,9 @@ def test_glob(self, pathlib_base): path_glob = list(pathlib_base.glob("**/*.txt")) _mock_start = len(self.path.parts) - mock_glob_normalized = sorted( - [a.parts[_mock_start:] for a in mock_glob] - ) + mock_glob_normalized = sorted([a.parts[_mock_start:] for a in mock_glob]) _path_start = len(pathlib_base.parts) - path_glob_normalized = sorted( - [a.parts[_path_start:] for a in path_glob] - ) + path_glob_normalized = sorted([a.parts[_path_start:] for a in path_glob]) print(mock_glob_normalized, path_glob_normalized) assert mock_glob_normalized == path_glob_normalized @@ -102,7 +99,7 @@ def test_iterdir(self, local_testdir): assert x.exists() assert len(up_iter) == len(pl_iter) - assert set(p.name for p in pl_iter) == set(u.name for u in up_iter) + assert {p.name for p in pl_iter} == {u.name for u in up_iter} assert next(self.path.parent.iterdir()).exists() def test_iterdir2(self, local_testdir): @@ -115,7 +112,7 @@ def test_iterdir2(self, local_testdir): assert x.exists() assert len(up_iter) == len(pl_iter) - assert set(p.name for p in pl_iter) == set(u.name for u in up_iter) + assert {p.name for p in pl_iter} == {u.name for u in up_iter} assert next(self.path.parent.iterdir()).exists() def test_parents(self): @@ -201,8 +198,7 @@ def test_read_bytes(self, pathlib_base): def test_read_text(self, local_testdir): upath = self.path.joinpath("file1.txt") assert ( - upath.read_text() - == Path(local_testdir).joinpath("file1.txt").read_text() + upath.read_text() == Path(local_testdir).joinpath("file1.txt").read_text() ) def test_readlink(self): diff --git a/upath/tests/conftest.py b/upath/tests/conftest.py index 9e5964e6..180c086b 100644 --- a/upath/tests/conftest.py +++ b/upath/tests/conftest.py @@ -1,18 +1,18 @@ import os +import shlex import shutil -import tempfile -from pathlib import Path import subprocess -import shlex +import sys +import tempfile import threading import time -import sys +from pathlib import Path +import fsspec import pytest from fsspec.implementations.local import LocalFileSystem -from fsspec.registry import register_implementation, _registry - -import fsspec +from fsspec.registry import _registry +from fsspec.registry import register_implementation from .utils import posixify @@ -171,10 +171,10 @@ def s3_server(): timeout -= 0.1 # pragma: no cover time.sleep(0.1) # pragma: no cover anon = False - s3so = dict( - client_kwargs={"endpoint_url": endpoint_uri}, - use_listings_cache=True, - ) + s3so = { + "client_kwargs": {"endpoint_url": endpoint_uri}, + "use_listings_cache": True, + } yield anon, s3so finally: proc.terminate() @@ -301,8 +301,8 @@ def http_fixture(local_testdir, http_server): @pytest.fixture(scope="session") def webdav_server(tmp_path_factory): try: - from wsgidav.wsgidav_app import WsgiDAVApp from cheroot import wsgi + from wsgidav.wsgidav_app import WsgiDAVApp except ImportError as err: pytest.skip(str(err)) @@ -315,9 +315,7 @@ def webdav_server(tmp_path_factory): "host": host, "port": port, "provider_mapping": {"/": tempdir}, - "simple_dc": { - "user_mapping": {"*": {"USER": {"password": "PASSWORD"}}} - }, + "simple_dc": {"user_mapping": {"*": {"USER": {"password": "PASSWORD"}}}}, } ) srvr = wsgi.Server(bind_addr=(host, port), wsgi_app=app) @@ -359,6 +357,9 @@ def azurite_credentials(): def docker_azurite(azurite_credentials): requests = pytest.importorskip("requests") + if shutil.which("docker") is None: + pytest.skip("docker not installed") + image = "mcr.microsoft.com/azure-storage/azurite" container_name = "azure_test" cmd = ( diff --git a/upath/tests/implementations/test_azure.py b/upath/tests/implementations/test_azure.py index 1d3c8a47..8c521536 100644 --- a/upath/tests/implementations/test_azure.py +++ b/upath/tests/implementations/test_azure.py @@ -1,4 +1,5 @@ import pytest + from upath import UPath from upath.implementations.cloud import AzurePath diff --git a/upath/tests/implementations/test_gcs.py b/upath/tests/implementations/test_gcs.py index ecfd7d1a..f72eeae8 100644 --- a/upath/tests/implementations/test_gcs.py +++ b/upath/tests/implementations/test_gcs.py @@ -2,6 +2,7 @@ from upath import UPath from upath.implementations.cloud import GCSPath + from ..cases import BaseTests from ..utils import skip_on_windows diff --git a/upath/tests/implementations/test_hdfs.py b/upath/tests/implementations/test_hdfs.py index 9bcc309d..9ba8ebae 100644 --- a/upath/tests/implementations/test_hdfs.py +++ b/upath/tests/implementations/test_hdfs.py @@ -4,6 +4,7 @@ from upath import UPath from upath.implementations.hdfs import HDFSPath + from ..cases import BaseTests diff --git a/upath/tests/implementations/test_http.py b/upath/tests/implementations/test_http.py index 6552ffcd..8bcc5ccb 100644 --- a/upath/tests/implementations/test_http.py +++ b/upath/tests/implementations/test_http.py @@ -1,9 +1,9 @@ import pytest # noqa: F401 - from fsspec import get_filesystem_class from upath import UPath from upath.implementations.http import HTTPPath + from ..cases import BaseTests from ..utils import skip_on_windows diff --git a/upath/tests/implementations/test_memory.py b/upath/tests/implementations/test_memory.py index eeea03ce..6a87df0c 100644 --- a/upath/tests/implementations/test_memory.py +++ b/upath/tests/implementations/test_memory.py @@ -2,6 +2,7 @@ from upath import UPath from upath.implementations.memory import MemoryPath + from ..cases import BaseTests diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index 5b042601..de9ef639 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -1,10 +1,11 @@ """see upath/tests/conftest.py for fixtures """ +import fsspec import pytest # noqa: F401 -import fsspec from upath import UPath from upath.implementations.cloud import S3Path + from ..cases import BaseTests @@ -36,16 +37,12 @@ def test_rmdir(self): def test_relative_to(self): assert "s3://test_bucket/file.txt" == str( - UPath("s3://test_bucket/file.txt").relative_to( - UPath("s3://test_bucket") - ) + UPath("s3://test_bucket/file.txt").relative_to(UPath("s3://test_bucket")) ) def test_iterdir_root(self): client_kwargs = self.path._kwargs["client_kwargs"] - bucket_path = UPath( - "s3://other_test_bucket", client_kwargs=client_kwargs - ) + bucket_path = UPath("s3://other_test_bucket", client_kwargs=client_kwargs) bucket_path.mkdir(mode="private") (bucket_path / "test1.txt").touch() @@ -69,9 +66,7 @@ def test_touch_unlink(self): # file doesn't exists, but missing_ok is True path.unlink(missing_ok=True) - @pytest.mark.parametrize( - "joiner", [["bucket", "path", "file"], "bucket/path/file"] - ) + @pytest.mark.parametrize("joiner", [["bucket", "path", "file"], "bucket/path/file"]) def test_no_bucket_joinpath(self, joiner): path = UPath("s3://", anon=self.anon, **self.s3so) path = path.joinpath(joiner) diff --git a/upath/tests/implementations/test_webdav.py b/upath/tests/implementations/test_webdav.py index 5ef17d82..0b534112 100644 --- a/upath/tests/implementations/test_webdav.py +++ b/upath/tests/implementations/test_webdav.py @@ -1,6 +1,7 @@ import pytest # noqa: F401 from upath import UPath + from ..cases import BaseTests diff --git a/upath/tests/test_core.py b/upath/tests/test_core.py index 6a2adf55..6dbf581e 100644 --- a/upath/tests/test_core.py +++ b/upath/tests/test_core.py @@ -4,10 +4,14 @@ import warnings import pytest + from upath import UPath -from upath.implementations.cloud import S3Path, GCSPath +from upath.implementations.cloud import GCSPath +from upath.implementations.cloud import S3Path + from .cases import BaseTests -from .utils import only_on_windows, skip_on_windows +from .utils import only_on_windows +from .utils import skip_on_windows @skip_on_windows @@ -40,7 +44,7 @@ def path(self, local_testdir): with warnings.catch_warnings(): warnings.simplefilter("ignore") - # On Windows the path needs to be prefixed with `/`, becaue + # On Windows the path needs to be prefixed with `/`, because # `UPath` implements `_posix_flavour`, which requires a `/` root # in order to correctly deserialize pickled objects root = "/" if sys.platform.startswith("win") else "" @@ -145,7 +149,7 @@ def test_create_from_type(path, storage_options, module, object_type): new = cast(str(parent)) # test that object cast is same type assert isinstance(new, cast) - except (ImportError, ModuleNotFoundError): + except ImportError: # fs failed to import pass @@ -248,15 +252,11 @@ def test_copy_path_append_kwargs(): def test_relative_to(): assert "s3://test_bucket/file.txt" == str( - UPath("s3://test_bucket/file.txt").relative_to( - UPath("s3://test_bucket") - ) + UPath("s3://test_bucket/file.txt").relative_to(UPath("s3://test_bucket")) ) with pytest.raises(ValueError): - UPath("s3://test_bucket/file.txt").relative_to( - UPath("gcs://test_bucket") - ) + UPath("s3://test_bucket/file.txt").relative_to(UPath("gcs://test_bucket")) with pytest.raises(ValueError): UPath("s3://test_bucket/file.txt", anon=True).relative_to( @@ -266,8 +266,7 @@ def test_relative_to(): def test_uri_parsing(): assert ( - str(UPath("http://www.example.com//a//b/")) - == "http://www.example.com//a//b/" + str(UPath("http://www.example.com//a//b/")) == "http://www.example.com//a//b/" ) diff --git a/upath/tests/utils.py b/upath/tests/utils.py index f356748e..62c9e0c9 100644 --- a/upath/tests/utils.py +++ b/upath/tests/utils.py @@ -1,4 +1,5 @@ import sys + import pytest