diff --git a/README.rst b/README.rst index 88b9c619..57d6b23b 100644 --- a/README.rst +++ b/README.rst @@ -5,10 +5,18 @@ audbackend |tests| |coverage| |docs| |python-versions| |license| Manage file storage on different backends. -At the moment we support: -* Artifactory_ with ``audbackend.Artifactory`` -* local file system with ``audbackend.FileSystem`` +At the moment we support +the following backends: + +* Artifactory_ with ``audbackend.backend.Artifactory`` +* local file system with ``audbackend.backend.FileSystem`` + +And the following interfaces +to access files on a backend: + +* unversioned with ``audbackend.interface.Unversioned`` +* versioned with ``audbackend.interface.Versioned`` Have a look at the installation_ instructions. diff --git a/audbackend/__init__.py b/audbackend/__init__.py index da645b15..5b2d7266 100644 --- a/audbackend/__init__.py +++ b/audbackend/__init__.py @@ -1,17 +1,18 @@ +from audbackend import backend from audbackend import interface from audbackend.core.api import access from audbackend.core.api import available from audbackend.core.api import create from audbackend.core.api import delete from audbackend.core.api import register -from audbackend.core.backend import Backend +from audbackend.core.backend.base import Base as Backend # legacy +from audbackend.core.backend.filesystem import FileSystem # legacy from audbackend.core.errors import BackendError -from audbackend.core.filesystem import FileSystem from audbackend.core.repository import Repository -# Import optional backends +# Import optional backends (legacy) try: - from audbackend.core.artifactory import Artifactory + from audbackend.core.backend.artifactory import Artifactory except ImportError: # pragma: no cover pass diff --git a/audbackend/backend/__init__.py b/audbackend/backend/__init__.py new file mode 100644 index 00000000..06be3a6a --- /dev/null +++ b/audbackend/backend/__init__.py @@ -0,0 +1,8 @@ +from audbackend.core.backend.base import Base +from audbackend.core.backend.filesystem import FileSystem + +# Import optional backends +try: + from audbackend.core.backend.artifactory import Artifactory +except ImportError: # pragma: no cover + pass diff --git a/audbackend/core/api.py b/audbackend/core/api.py index 75c708e1..490c7e77 100644 --- a/audbackend/core/api.py +++ b/audbackend/core/api.py @@ -1,8 +1,8 @@ import typing from audbackend.core import utils -from audbackend.core.backend import Backend -from audbackend.core.filesystem import FileSystem +from audbackend.core.backend.base import Base +from audbackend.core.backend.filesystem import FileSystem from audbackend.core.interface.base import Base as Interface from audbackend.core.interface.versioned import Versioned @@ -18,7 +18,7 @@ def _backend( name: str, host: str, repository: str, -) -> Backend: +) -> Base: r"""Get backend instance.""" if name not in backend_registry: raise ValueError( @@ -89,7 +89,7 @@ def access( Examples: >>> access('file-system', 'host', 'repo') - audbackend.core.interface.versioned.Versioned('audbackend.core.filesystem.FileSystem', 'host', 'repo') + audbackend.core.interface.versioned.Versioned('audbackend.core.backend.filesystem.FileSystem', 'host', 'repo') """ # noqa: E501 backend = _backend(name, host, repository) @@ -98,7 +98,7 @@ def access( return interface(backend, **interface_kwargs) -def available() -> typing.Dict[str, typing.List[Backend]]: +def available() -> typing.Dict[str, typing.List[Base]]: r"""List available backend instances. Returns a dictionary with @@ -114,7 +114,7 @@ def available() -> typing.Dict[str, typing.List[Backend]]: >>> list(available()) ['artifactory', 'file-system'] >>> available()['file-system'][0] - ('audbackend.core.filesystem.FileSystem', 'host', 'repo') + ('audbackend.core.backend.filesystem.FileSystem', 'host', 'repo') """ # noqa: E501 result = {} @@ -174,7 +174,7 @@ def create( Examples: >>> create('file-system', 'host', 'repository') - audbackend.core.interface.versioned.Versioned('audbackend.core.filesystem.FileSystem', 'host', 'repository') + audbackend.core.interface.versioned.Versioned('audbackend.core.backend.filesystem.FileSystem', 'host', 'repository') """ # noqa: E501 backend = _backend(name, host, repository) @@ -222,7 +222,7 @@ def delete( def register( name: str, - cls: typing.Type[Backend], + cls: typing.Type[Base], ): r"""Register backend class. @@ -245,7 +245,7 @@ def register( # Register optional backends try: - from audbackend.core.artifactory import Artifactory + from audbackend.core.backend.artifactory import Artifactory register('artifactory', Artifactory) except ImportError: # pragma: no cover pass diff --git a/audbackend/core/backend/__init__.py b/audbackend/core/backend/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/audbackend/core/artifactory.py b/audbackend/core/backend/artifactory.py similarity index 99% rename from audbackend/core/artifactory.py rename to audbackend/core/backend/artifactory.py index a9f9163d..910d39d4 100644 --- a/audbackend/core/artifactory.py +++ b/audbackend/core/backend/artifactory.py @@ -7,7 +7,7 @@ import audeer from audbackend.core import utils -from audbackend.core.backend import Backend +from audbackend.core.backend.base import Base def _artifactory_path( @@ -110,7 +110,7 @@ def _download( pbar.update(n_data) -class Artifactory(Backend): +class Artifactory(Base): r"""Backend for Artifactory. Looks for the two environment variables diff --git a/audbackend/core/backend.py b/audbackend/core/backend/base.py similarity index 99% rename from audbackend/core/backend.py rename to audbackend/core/backend/base.py index 0d5684ba..85f15aec 100644 --- a/audbackend/core/backend.py +++ b/audbackend/core/backend/base.py @@ -9,7 +9,7 @@ from audbackend.core.errors import BackendError -class Backend: +class Base: r"""Backend base class. Derive from this class to implement a new backend. diff --git a/audbackend/core/filesystem.py b/audbackend/core/backend/filesystem.py similarity index 98% rename from audbackend/core/filesystem.py rename to audbackend/core/backend/filesystem.py index 7f5f3ba2..5741af02 100644 --- a/audbackend/core/filesystem.py +++ b/audbackend/core/backend/filesystem.py @@ -6,10 +6,10 @@ import audeer from audbackend.core import utils -from audbackend.core.backend import Backend +from audbackend.core.backend.base import Base -class FileSystem(Backend): +class FileSystem(Base): r"""Backend for file system. Args: diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 6992159d..959914a1 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -9,10 +9,10 @@ import audbackend -class DoctestFileSystem(audbackend.FileSystem): +class DoctestFileSystem(audbackend.backend.FileSystem): def __repr__(self) -> str: - name = 'audbackend.core.filesystem.FileSystem' + name = 'audbackend.core.backend.filesystem.FileSystem' return str((name, self.host, self.repository)) def _date( @@ -45,7 +45,7 @@ def prepare_docstring_tests(doctest_namespace): # backend - backend = audbackend.Backend('host', 'repo') + backend = audbackend.backend.Base('host', 'repo') doctest_namespace['backend'] = backend interface = audbackend.interface.Base(backend) @@ -84,6 +84,6 @@ def prepare_docstring_tests(doctest_namespace): audbackend.delete('file-system', 'host', 'repo') audbackend.delete('file-system', 'host', 'repo-unversioned') - audbackend.register('file-system', audbackend.FileSystem) + audbackend.register('file-system', audbackend.backend.FileSystem) os.chdir(current_dir) diff --git a/audbackend/core/interface/__init__.py b/audbackend/core/interface/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/audbackend/core/interface/base.py b/audbackend/core/interface/base.py index 657cf90f..1e3315c6 100644 --- a/audbackend/core/interface/base.py +++ b/audbackend/core/interface/base.py @@ -1,6 +1,6 @@ import typing -from audbackend.core.backend import Backend +from audbackend.core.backend.base import Base as Backend class Base: @@ -38,7 +38,7 @@ def backend(self) -> Backend: Examples: >>> interface.backend - ('audbackend.core.backend.Backend', 'host', 'repo') + ('audbackend.core.backend.base.Base', 'host', 'repo') """ return self._backend diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py index c10b5747..907400f5 100644 --- a/audbackend/core/interface/versioned.py +++ b/audbackend/core/interface/versioned.py @@ -5,7 +5,6 @@ import audeer from audbackend.core import utils -from audbackend.core.backend import Backend from audbackend.core.errors import BackendError from audbackend.core.interface.base import Base @@ -20,7 +19,7 @@ class Versioned(Base): def __init__( self, - backend: Backend, + backend: Base, ): super().__init__(backend) diff --git a/docs/api-src/audbackend.backend.rst b/docs/api-src/audbackend.backend.rst new file mode 100644 index 00000000..8ddd01b0 --- /dev/null +++ b/docs/api-src/audbackend.backend.rst @@ -0,0 +1,24 @@ +audbackend.backend +================== + +.. automodule:: audbackend.backend + +Currently the following +backends are supported: + +.. autosummary:: + :toctree: + :nosignatures: + + Artifactory + FileSystem + +Users can implement their own +backend by deriving from +:class:`audbackend.backend.Base`. + +.. autosummary:: + :toctree: + :nosignatures: + + Base diff --git a/docs/api-src/audbackend.interface.rst b/docs/api-src/audbackend.interface.rst index b59e74ed..5df85253 100644 --- a/docs/api-src/audbackend.interface.rst +++ b/docs/api-src/audbackend.interface.rst @@ -4,10 +4,7 @@ audbackend.interface .. automodule:: audbackend.interface To access the files on a backend users -can choose between two interfaces -:class:`audbackend.interface.Unversioned` -or -:class:`audbackend.interface.Versioned`: +can use the following interfaces: .. autosummary:: :toctree: diff --git a/docs/api-src/audbackend.rst b/docs/api-src/audbackend.rst index cc942d50..f9705094 100644 --- a/docs/api-src/audbackend.rst +++ b/docs/api-src/audbackend.rst @@ -3,32 +3,26 @@ audbackend .. automodule:: audbackend -A backend is a generic interface +:mod:`audbackend` +provides an abstract layer for storing and accessing files on a host. -Currently the following backends -are shipped with :mod:`audbackend`: +This involves two components: -.. autosummary:: - :toctree: - :nosignatures: - - Artifactory - FileSystem - -Users can implement their own -backend by deriving from -:class:`audbackend.Backend`. - -.. autosummary:: - :toctree: - :nosignatures: +1. A *backend* that + implements file operations + on a specific storing device + (:mod:`audbackend.backend`). - Backend +2. An *interface* that + passes user requests + to a backend + (:mod:`audbackend.interface`). -In addition to the backends -the following classes and functions are available. +Additionally, +the following classes +and functions are available. .. autosummary:: :toctree: diff --git a/docs/index.rst b/docs/index.rst index 9070cdaf..b520c79a 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ :hidden: api/audbackend + api/audbackend.backend api/audbackend.interface genindex diff --git a/docs/install.rst b/docs/install.rst index 32562d46..d0911b32 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -11,7 +11,8 @@ To install :mod:`audbackend` run: $ pip install audbackend By default, -only the :class:`audbackend.FileSystem` backend will be installed. +only the :class:`audbackend.backend.FileSystem` +backend will be installed. To install all backends run: .. code-block:: bash @@ -19,7 +20,7 @@ To install all backends run: $ pip install audbackend[all] or select single backends, -e.g. :class:`audbackend.Artifactory` +e.g. :class:`audbackend.backend.Artifactory` .. code-block:: bash diff --git a/docs/usage.rst b/docs/usage.rst index 8f85dda5..3a5addee 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -42,17 +42,10 @@ a SQLite_ database. File-system example ------------------- -The heart of -:mod:`audbackend` -is the class -:class:`audbackend.Backend`, -which provides an abstract -interface to communicate -with a storage system. The class -:class:`audbackend.FileSystem` +:class:`audbackend.backend.FileSystem` implements -:class:`audbackend.Backend` +:class:`audbackend.backend.Base` for a standard file system. @@ -67,7 +60,7 @@ if we do it again). import audbackend - audbackend.register('file-system', audbackend.FileSystem) + audbackend.register('file-system', audbackend.backend.FileSystem) To make sure we can keep track @@ -199,8 +192,8 @@ on the backend. When we get an archive from the backend we can automatically extract it, -by using :meth:`audbackend.FileSystem.get_archive` -instead of :meth:`audbackend.FileSystem.get_file`. +by using :meth:`audbackend.backend.FileSystem.get_archive` +instead of :meth:`audbackend.backend.FileSystem.get_file`. .. jupyter-execute:: @@ -349,7 +342,7 @@ helper class. and write changes back to the backend. """ - def __init__(self, backend: audbackend.Backend): + def __init__(self, backend: audbackend.backend.Base): self.backend = backend def __enter__(self) -> shelve.Shelf: @@ -420,7 +413,7 @@ a SQLite_ database. A new backend should be implemented as a class deriving from -:class:`audbackend.Backend`. +:class:`audbackend.backend.Base`. As can be seen in the file :file:`audbackend/core/backend.py`, we need to implement the following private methods: @@ -451,7 +444,7 @@ in the constructor: import audbackend import os - class SQLite(audbackend.Backend): + class SQLite(audbackend.backend.Base): def __init__( self, diff --git a/tests/conftest.py b/tests/conftest.py index 69c9d398..87861d9d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,7 @@ import audbackend -# list of backends that will be tested by default +# list of backends to test pytest.BACKENDS = [ 'artifactory', 'file-system', @@ -19,12 +19,13 @@ pytest.BACKENDS.append('single-folder') +# list of interfaces to test pytest.UNVERSIONED = [ (backend, audbackend.interface.Unversioned) for backend in pytest.BACKENDS ] pytest.VERSIONED = [ - ('file-system', audbackend.interface.Versioned) + ('file-system', audbackend.interface.Versioned), ] # UID for test session @@ -53,7 +54,7 @@ def owner(request): r"""Return expected owner value.""" name = request.param if name == 'artifactory': - owner = audbackend.core.artifactory._authentication( + owner = audbackend.core.backend.artifactory._authentication( 'audeering.jfrog.io/artifactory' )[0] else: diff --git a/tests/singlefolder.py b/tests/singlefolder.py index a85ae0d2..c9354458 100644 --- a/tests/singlefolder.py +++ b/tests/singlefolder.py @@ -10,7 +10,7 @@ import audbackend -class SingleFolder(audbackend.Backend): +class SingleFolder(audbackend.backend.Base): r"""Backend implemented in a single folder. Files put on the backend diff --git a/tests/test_api.py b/tests/test_api.py index 986d3edd..38da83f2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -12,13 +12,13 @@ 'file-system', 'file-system', f'unittest-{audeer.uid()[:8]}', - audbackend.FileSystem, + audbackend.backend.FileSystem, ), ( 'artifactory', 'artifactory', f'unittest-{audeer.uid()[:8]}', - audbackend.Artifactory, + audbackend.backend.Artifactory, ), pytest.param( # backend does not exist 'bad-backend', diff --git a/tests/test_backend_artifactory.py b/tests/test_backend_artifactory.py index 863adb7e..5497b66b 100644 --- a/tests/test_backend_artifactory.py +++ b/tests/test_backend_artifactory.py @@ -41,14 +41,14 @@ def test_authentication(tmpdir, hosts, hide_credentials): # config file does not exist - backend = audbackend.Artifactory(host, 'repository') + backend = audbackend.backend.Artifactory(host, 'repository') assert backend._username == 'anonymous' assert backend._api_key == '' # config file is empty audeer.touch(config_path) - backend = audbackend.Artifactory(host, 'repository') + backend = audbackend.backend.Artifactory(host, 'repository') assert backend._username == 'anonymous' assert backend._api_key == '' @@ -57,7 +57,7 @@ def test_authentication(tmpdir, hosts, hide_credentials): with open(config_path, 'w') as fp: fp.write(f'[{host}]\n') - backend = audbackend.Artifactory(host, 'repository') + backend = audbackend.backend.Artifactory(host, 'repository') assert backend._username == 'anonymous' assert backend._api_key == '' @@ -71,7 +71,7 @@ def test_authentication(tmpdir, hosts, hide_credentials): fp.write(f'password = {api_key}\n') with pytest.raises(dohq_artifactory.exception.ArtifactoryException): - audbackend.Artifactory(host, 'repository') + audbackend.backend.Artifactory(host, 'repository') @pytest.mark.parametrize( diff --git a/tests/test_backend_base.py b/tests/test_backend_base.py index 1157ffbe..f8ee6422 100644 --- a/tests/test_backend_base.py +++ b/tests/test_backend_base.py @@ -40,7 +40,7 @@ @pytest.mark.parametrize( 'backend', [ - audbackend.Backend('host', 'repository'), + audbackend.backend.Base('host', 'repository'), ] ) def test_join(paths, expected, backend): @@ -76,7 +76,7 @@ def test_join(paths, expected, backend): @pytest.mark.parametrize( 'backend', [ - audbackend.Backend('host', 'repository'), + audbackend.backend.Base('host', 'repository'), ] ) def test_split(path, expected, backend): diff --git a/tests/test_backend_filesystem.py b/tests/test_backend_filesystem.py index d0097b79..17b7cd9a 100644 --- a/tests/test_backend_filesystem.py +++ b/tests/test_backend_filesystem.py @@ -7,7 +7,7 @@ import audbackend -class BadFileSystem(audbackend.FileSystem): +class BadFileSystem(audbackend.backend.FileSystem): r"""Imitates a corrupted file system.""" def _get_file( self, @@ -24,7 +24,7 @@ def _get_file( def bad_file_system(): audbackend.register('file-system', BadFileSystem) yield - audbackend.register('file-system', audbackend.FileSystem) + audbackend.register('file-system', audbackend.backend.FileSystem) @pytest.mark.parametrize( diff --git a/tests/test_backend_filesystem_only.py b/tests/test_backend_filesystem_only.py index 12617250..23db6719 100644 --- a/tests/test_backend_filesystem_only.py +++ b/tests/test_backend_filesystem_only.py @@ -5,4 +5,4 @@ # Check optional backends are not available with pytest.raises(AttributeError): - audbackend.Artifactory('https://host.com', 'repo') + audbackend.backend.Artifactory('https://host.com', 'repo') diff --git a/tests/test_legacy_import.py b/tests/test_legacy_import.py new file mode 100644 index 00000000..c3a09453 --- /dev/null +++ b/tests/test_legacy_import.py @@ -0,0 +1,8 @@ +import audbackend + + +def test_legacy_import(hosts): + + audbackend.Backend('host', 'repo') + audbackend.Artifactory(hosts['artifactory'], 'repo') + audbackend.FileSystem(hosts['file-system'], 'repo')