diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc2bd66a..44141421 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,7 +40,7 @@ jobs: - name: Test file-system only package run: | - python -m pytest --cov-fail-under=0 tests/test_filesystem_only.py tests/test_filesystem.py + python -m pytest --cov-fail-under=0 tests/test_backend_filesystem_only.py tests/test_backend_filesystem.py - name: Install package with all dependencies run: | diff --git a/audbackend/__init__.py b/audbackend/__init__.py index ecafd89e..da645b15 100644 --- a/audbackend/__init__.py +++ b/audbackend/__init__.py @@ -1,3 +1,4 @@ +from audbackend import interface from audbackend.core.api import access from audbackend.core.api import available from audbackend.core.api import create diff --git a/audbackend/core/api.py b/audbackend/core/api.py index 6bdfc682..75c708e1 100644 --- a/audbackend/core/api.py +++ b/audbackend/core/api.py @@ -3,6 +3,8 @@ from audbackend.core import utils from audbackend.core.backend import Backend from audbackend.core.filesystem import FileSystem +from audbackend.core.interface.base import Base as Interface +from audbackend.core.interface.versioned import Versioned backends = {} @@ -31,26 +33,31 @@ def _backend( if host not in backends[name]: backends[name][host] = {} if repository not in backends[name][host]: - backends[name][host][repository] = utils.call_function_on_backend( + backend = utils.call_function_on_backend( backend_registry[name], host, repository, ) + backends[name][host][repository] = backend - return backends[name][host][repository] + backend = backends[name][host][repository] + return backend def access( name: str, host: str, repository: str, -) -> Backend: + *, + interface: typing.Type[Interface] = Versioned, + interface_kwargs: dict = None, +) -> Interface: r"""Access repository. - Returns a backend instance - for the ``repository`` + Returns an ``interface`` instance + to access the ``repository`` on the ``host``. - The instance is an object of the class + The backend is an object of the class registered under the alias ``name`` with :func:`audbackend.register`. @@ -68,9 +75,11 @@ def access( name: alias under which backend class is registered host: host address repository: repository name + interface: interface class + interface_kwargs: keyword arguments for interface class Returns: - backend object + interface object Raises: BackendError: if an error is raised on the backend, @@ -79,13 +88,14 @@ def access( has been registered Examples: - >>> access('file-system', 'host', 'doctest') - ('audbackend.core.filesystem.FileSystem', 'host', 'doctest') + >>> access('file-system', 'host', 'repo') + audbackend.core.interface.versioned.Versioned('audbackend.core.filesystem.FileSystem', 'host', 'repo') - """ + """ # noqa: E501 backend = _backend(name, host, repository) utils.call_function_on_backend(backend._access) - return backend + interface_kwargs = interface_kwargs or {} + return interface(backend, **interface_kwargs) def available() -> typing.Dict[str, typing.List[Backend]]: @@ -103,8 +113,8 @@ def available() -> typing.Dict[str, typing.List[Backend]]: Examples: >>> list(available()) ['artifactory', 'file-system'] - >>> available()['file-system'] - [('audbackend.core.filesystem.FileSystem', 'host', 'doctest')] + >>> available()['file-system'][0] + ('audbackend.core.filesystem.FileSystem', 'host', 'repo') """ # noqa: E501 result = {} @@ -123,13 +133,15 @@ def create( name: str, host: str, repository: str, -) -> Backend: + *, + interface: typing.Type[Interface] = Versioned, + interface_kwargs: dict = None +) -> Interface: r"""Create repository. - Creates the ``repository`` - on the ``host`` - and returns a backend instance for it. - The instance is an object of the class + Creates ``repository`` on the ``host`` + and returns an ``interface`` instance for it. + The backend is an object of the class registered under the alias ``name`` with :func:`audbackend.register`. @@ -147,9 +159,11 @@ def create( name: alias under which backend class is registered host: host address repository: repository name + interface: interface class + interface_kwargs: keyword arguments for interface class Returns: - backend object + interface object Raises: BackendError: if an error is raised on the backend, @@ -160,12 +174,13 @@ def create( Examples: >>> create('file-system', 'host', 'repository') - ('audbackend.core.filesystem.FileSystem', 'host', 'repository') + audbackend.core.interface.versioned.Versioned('audbackend.core.filesystem.FileSystem', 'host', 'repository') - """ + """ # noqa: E501 backend = _backend(name, host, repository) utils.call_function_on_backend(backend._create) - return backend + interface_kwargs = interface_kwargs or {} + return interface(backend, **interface_kwargs) def delete( @@ -193,15 +208,15 @@ def delete( has been registered Examples: - >>> access('file-system', 'host', 'doctest').ls() + >>> access('file-system', 'host', 'repo').ls() [('/a.zip', '1.0.0'), ('/a/b.ext', '1.0.0'), ('/f.ext', '1.0.0'), ('/f.ext', '2.0.0')] - >>> delete('file-system', 'host', 'doctest') - >>> create('file-system', 'host', 'doctest').ls() + >>> delete('file-system', 'host', 'repo') + >>> create('file-system', 'host', 'repo').ls() [] """ # noqa: E501 - backend = access(name, host, repository) - utils.call_function_on_backend(backend._delete) + interface = access(name, host, repository) + utils.call_function_on_backend(interface._backend._delete) backends[name][host].pop(repository) diff --git a/audbackend/core/backend.py b/audbackend/core/backend.py index 400972e4..0d5684ba 100644 --- a/audbackend/core/backend.py +++ b/audbackend/core/backend.py @@ -1,7 +1,5 @@ -import errno import fnmatch import os -import re import tempfile import typing @@ -12,13 +10,9 @@ class Backend: - r"""Abstract backend. + r"""Backend base class. - A backend stores files and archives. - - Args: - host: host address - repository: repository name + Derive from this class to implement a new backend. """ def __init__( @@ -31,12 +25,6 @@ def __init__( self.repository = repository r"""Repository name.""" - # to support legacy file structure - # see _use_legacy_file_structure() - self._legacy_extensions = [] - self._legacy_file_structure = False - self._legacy_file_structure_regex = False - def __repr__(self) -> str: # noqa: D105 name = f'{self.__class__.__module__}.{self.__class__.__name__}' return str((name, self.host, self.repository)) @@ -61,13 +49,11 @@ def _checksum( def checksum( self, path: str, - version: str, ) -> str: - r"""Get MD5 checksum for file on backend. + r"""MD5 checksum for file on backend. Args: path: path to file on backend - version: version string Returns: MD5 checksum @@ -77,19 +63,12 @@ def checksum( e.g. ``path`` does not exist ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> backend.checksum('/f.ext', '1.0.0') - 'd41d8cd98f00b204e9800998ecf8427e' """ - path_with_version = self._path_with_version(path, version) - + path = utils.check_path(path) return utils.call_function_on_backend( self._checksum, - path_with_version, + path, ) def _create( @@ -106,7 +85,7 @@ def _date( self, path: str, ) -> str: # pragma: no cover - r"""Get date of file on backend. + r"""Last modification date of file on backend. * Return empty string if date cannot be determined * Format should be '%Y-%m-%d' @@ -117,16 +96,14 @@ def _date( def date( self, path: str, - version: str, ) -> str: - r"""Get last modification date of file on backend. + r"""Last modification date of file on backend. If the date cannot be determined, an empty string is returned. Args: path: path to file on backend - version: version string Returns: date in format ``'yyyy-mm-dd'`` @@ -136,19 +113,12 @@ def date( e.g. ``path`` does not exist ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> backend.date('/f.ext', '1.0.0') - '1991-02-20' """ - path_with_version = self._path_with_version(path, version) - + path = utils.check_path(path) return utils.call_function_on_backend( self._date, - path_with_version, + path, ) def _delete( @@ -167,7 +137,6 @@ def _exists( def exists( self, path: str, - version: str, *, suppress_backend_errors: bool = False, ) -> bool: @@ -175,7 +144,6 @@ def exists( Args: path: path to file on backend - version: version string suppress_backend_errors: if set to ``True``, silently catch errors raised on the backend and return ``False`` @@ -192,16 +160,11 @@ def exists( ValueError: if ``version`` is empty or does not match ``'[A-Za-z0-9._-]+'`` - Examples: - >>> backend.exists('/f.ext', '1.0.0') - True - """ - path_with_version = self._path_with_version(path, version) - + path = utils.check_path(path) return utils.call_function_on_backend( self._exists, - path_with_version, + path, suppress_backend_errors=suppress_backend_errors, fallback_return_value=False, ) @@ -210,7 +173,6 @@ def get_archive( self, src_path: str, dst_root: str, - version: str, *, tmp_root: str = None, verbose: bool = False, @@ -226,7 +188,6 @@ def get_archive( Args: src_path: path to archive on backend dst_root: local destination directory - version: version string tmp_root: directory under which archive is temporarily extracted. Defaults to temporary directory of system verbose: show debug messages @@ -245,16 +206,9 @@ def get_archive( or ``src_path`` is a malformed archive ValueError: if ``src_path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> backend.get_archive('/a.zip', '.', '1.0.0') - ['src.pth'] """ src_path = utils.check_path(src_path) - version = utils.check_version(version) with tempfile.TemporaryDirectory(dir=tmp_root) as tmp: @@ -266,7 +220,6 @@ def get_archive( self.get_file( src_path, local_archive, - version, verbose=verbose, ) @@ -289,7 +242,6 @@ def get_file( self, src_path: str, dst_path: str, - version: str, *, verbose: bool = False, ) -> str: @@ -312,7 +264,6 @@ def get_file( Args: src_path: path to file on backend dst_path: destination path to local file - version: version string verbose: show debug messages Returns: @@ -326,19 +277,9 @@ def get_file( for ``dst_path`` ValueError: if ``src_path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> os.path.exists('dst.pth') - False - >>> _ = backend.get_file('/f.ext', 'dst.pth', '1.0.0') - >>> os.path.exists('dst.pth') - True """ - src_path_with_version = self._path_with_version(src_path, version) - + src_path = utils.check_path(src_path) dst_path = audeer.path(dst_path) if os.path.isdir(dst_path): raise utils.raise_is_a_directory(dst_path) @@ -355,7 +296,7 @@ def get_file( if ( not os.path.exists(dst_path) - or audeer.md5(dst_path) != self.checksum(src_path, version) + or audeer.md5(dst_path) != self.checksum(src_path) ): # get file to a temporary directory first, # only on success move to final destination @@ -363,7 +304,7 @@ def get_file( tmp_path = audeer.path(tmp, '~') utils.call_function_on_backend( self._get_file, - src_path_with_version, + src_path, tmp_path, verbose, ) @@ -390,14 +331,6 @@ def join( or does not start with ``'/'``, or if joined path contains invalid character - Examples: - >>> backend.join('/', 'f.ext') - '/f.ext' - >>> backend.join('/sub', 'f.ext') - '/sub/f.ext' - >>> backend.join('//sub//', '/', '', None, '/f.ext') - '/sub/f.ext' - """ path = utils.check_path(path) @@ -409,60 +342,6 @@ def join( return path - def latest_version( - self, - path: str, - ) -> str: - r"""Latest version of a file. - - Args: - path: path to file on backend - - Returns: - version string - - Raises: - BackendError: if an error is raised on the backend, - e.g. ``path`` does not exist - ValueError: if ``path`` does not start with ``'/'`` or - does not match ``'[A-Za-z0-9/._-]+'`` - - Examples: - >>> backend.latest_version('/f.ext') - '2.0.0' - - """ - vs = self.versions(path) - return vs[-1] - - def _legacy_split_ext( - self, - name: str, - ) -> typing.Tuple[str, str]: - r"""Split name into basename and extension.""" - ext = None - for custom_ext in self._legacy_extensions: - # check for custom extension - # ensure basename is not empty - if self._legacy_file_structure_regex: - pattern = rf'\.({custom_ext})$' - match = re.search(pattern, name[1:]) - if match: - ext = match.group(1) - elif name[1:].endswith(f'.{custom_ext}'): - ext = custom_ext - if ext is None: - # if no custom extension is found - # use last string after dot - ext = audeer.file_extension(name) - - base = audeer.replace_file_extension(name, '', ext=ext) - - if ext: - ext = f'.{ext}' - - return base, ext - def _ls( self, path: str, @@ -482,10 +361,9 @@ def ls( self, path: str = '/', *, - latest_version: bool = False, pattern: str = None, suppress_backend_errors: bool = False, - ) -> typing.List[typing.Tuple[str, str]]: + ) -> typing.List[str]: r"""List files on backend. Returns a sorted list of tuples @@ -507,8 +385,6 @@ def ls( path: path or sub-path (if it ends with ``'/'``) on backend - latest_version: if multiple versions of a file exist, - only include the latest pattern: if not ``None``, return only files matching the pattern string, see :func:`fnmatch.fnmatch` @@ -526,21 +402,7 @@ def ls( ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - Examples: - >>> backend.ls() - [('/a.zip', '1.0.0'), ('/a/b.ext', '1.0.0'), ('/f.ext', '1.0.0'), ('/f.ext', '2.0.0')] - >>> backend.ls(latest_version=True) - [('/a.zip', '1.0.0'), ('/a/b.ext', '1.0.0'), ('/f.ext', '2.0.0')] - >>> backend.ls('/f.ext') - [('/f.ext', '1.0.0'), ('/f.ext', '2.0.0')] - >>> backend.ls(pattern='*.ext') - [('/a/b.ext', '1.0.0'), ('/f.ext', '1.0.0'), ('/f.ext', '2.0.0')] - >>> backend.ls(pattern='b.*') - [('/a/b.ext', '1.0.0')] - >>> backend.ls('/a/') - [('/a/b.ext', '1.0.0')] - - """ # noqa: E501 + """ path = utils.check_path(path) if path.endswith('/'): # find files under sub-path @@ -552,101 +414,39 @@ def ls( fallback_return_value=[], ) - else: # find versions of path + else: # find path - root, file = self.split(path) - paths = utils.call_function_on_backend( - self._ls, - root, - suppress_backend_errors=suppress_backend_errors, - fallback_return_value=[], - ) - - # filter for '/root/version/file' - if self._legacy_file_structure: - depth = root.count('/') + 2 - name, ext = self._legacy_split_ext(file) - match = re.compile(rf'{name}-\d+\.\d+.\d+{ext}') - paths = [ - p for p in paths - if ( - p.count('/') == depth and - match.match(os.path.basename(p)) - ) - ] + if self.exists(path): + paths = [path] else: - depth = root.count('/') + 1 - paths = [ - p for p in paths - if ( - p.count('/') == depth and - os.path.basename(p) == file - ) - ] - - if not paths and not suppress_backend_errors: - # since the backend does no longer raise an error - # if the path does not exist - # we have to do it - ex = FileNotFoundError( - errno.ENOENT, - os.strerror(errno.ENOENT), - path, - ) - raise BackendError(ex) + if not suppress_backend_errors: + # since the backend does no longer raise an error + # if the path does not exist + # we have to do it + try: + raise utils.raise_file_not_found_error(path) + except FileNotFoundError as ex: + raise BackendError(ex) + paths = [] if not paths: return [] - paths_and_versions = [] - for p in paths: - - tokens = p.split(self.sep) - - name = tokens[-1] - version = tokens[-2] - - if self._legacy_file_structure: - base = tokens[-3] - ext = name[len(base) + len(version) + 1:] - name = f'{base}{ext}' - path = self.sep.join(tokens[:-3]) - else: - path = self.sep.join(tokens[:-2]) - - path = self.sep + path - path = self.join(path, name) - - paths_and_versions.append((path, version)) - - paths_and_versions = sorted(paths_and_versions) + paths = sorted(paths) if pattern: - paths_and_versions = [ - (p, v) for p, v in paths_and_versions + paths = [ + p for p in paths if fnmatch.fnmatch(os.path.basename(p), pattern) ] - if latest_version: - # d[path] = ['1.0.0', '2.0.0'] - d = {} - for p, v in paths_and_versions: - if p not in d: - d[p] = [] - d[p].append(v) - # d[path] = '2.0.0' - for p, vs in d.items(): - d[p] = audeer.sort_versions(vs)[-1] - # [(path, '2.0.0')] - paths_and_versions = [(p, v) for p, v in d.items()] - - return paths_and_versions + return paths def _owner( self, path: str, ) -> str: # pragma: no cover - r"""Get owner of file on backend. + r"""Owner of file on backend. * Return empty string if owner cannot be determined @@ -656,9 +456,8 @@ def _owner( def owner( self, path: str, - version: str, ) -> str: - r"""Get owner of file on backend. + r"""Owner of file on backend. If the owner of the file cannot be determined, @@ -666,7 +465,6 @@ def owner( Args: path: path to file on backend - version: version string Returns: owner @@ -676,57 +474,18 @@ def owner( e.g. ``path`` does not exist ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> backend.owner('/f.ext', '1.0.0') - 'doctest' """ - path_with_version = self._path_with_version(path, version) - + path = utils.check_path(path) return utils.call_function_on_backend( self._owner, - path_with_version, + path, ) - def _path_with_version( - self, - path: str, - version: str, - ) -> str: - r"""Convert to versioned path. - - / - -> - // - - or legacy: - - / - -> - ///- - - """ - path = utils.check_path(path) - version = utils.check_version(version) - - root, name = self.split(path) - - if self._legacy_file_structure: - base, ext = self._legacy_split_ext(name) - path = self.join(root, base, version, f'{base}-{version}{ext}') - else: - path = self.join(root, version, name) - - return path - def put_archive( self, src_root: str, dst_path: str, - version: str, *, files: typing.Union[str, typing.Sequence[str]] = None, tmp_root: str = None, @@ -748,37 +507,14 @@ def put_archive( will be included into the archive. Use ``files`` to select specific files dst_path: path to archive on backend - version: version string files: file(s) to include into the archive. Must exist within ``src_root`` tmp_root: directory under which archive is temporarily created. Defaults to temporary directory of system verbose: show debug messages - Raises: - BackendError: if an error is raised on the backend - FileNotFoundError: if ``src_root``, - ``tmp_root``, - or one or more ``files`` do not exist - NotADirectoryError: if ``src_root`` is not a folder - RuntimeError: if ``dst_path`` does not end with - ``zip`` or ``tar.gz`` - or a file in ``files`` is not below ``root`` - ValueError: if ``dst_path`` does not start with ``'/'`` or - does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> backend.exists('/a.tar.gz', '1.0.0') - False - >>> backend.put_archive('.', '/a.tar.gz', '1.0.0') - >>> backend.exists('/a.tar.gz', '1.0.0') - True - """ dst_path = utils.check_path(dst_path) - version = utils.check_version(version) src_root = audeer.path(src_root) if tmp_root is not None: @@ -799,7 +535,6 @@ def put_archive( self.put_file( archive, dst_path, - version, verbose=verbose, ) @@ -817,7 +552,6 @@ def put_file( self, src_path: str, dst_path: str, - version: str, *, verbose: bool = False, ): @@ -830,7 +564,6 @@ def put_file( Args: src_path: path to local file dst_path: path to file on backend - version: version string verbose: show debug messages Returns: @@ -842,19 +575,9 @@ def put_file( IsADirectoryError: if ``src_path`` is a folder ValueError: if ``dst_path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> backend.exists('/sub/f.ext', '3.0.0') - False - >>> backend.put_file('src.pth', '/sub/f.ext', '3.0.0') - >>> backend.exists('/sub/f.ext', '3.0.0') - True """ - dst_path_with_version = self._path_with_version(dst_path, version) - + dst_path = utils.check_path(dst_path) if not os.path.exists(src_path): utils.raise_file_not_found_error(src_path) elif os.path.isdir(src_path): @@ -864,13 +587,13 @@ def put_file( # skip if file with same checksum already exists if ( - not self.exists(dst_path, version) - or self.checksum(dst_path, version) != checksum + not self.exists(dst_path) + or self.checksum(dst_path) != checksum ): utils.call_function_on_backend( self._put_file, src_path, - dst_path_with_version, + dst_path, checksum, verbose, ) @@ -885,40 +608,32 @@ def _remove_file( def remove_file( self, path: str, - version: str, ): r"""Remove file from backend. Args: path: path to file on backend - version: version string Raises: BackendError: if an error is raised on the backend, e.g. ``path`` does not exist ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - ValueError: if ``version`` is empty or - does not match ``'[A-Za-z0-9._-]+'`` - - Examples: - >>> backend.exists('/f.ext', '1.0.0') - True - >>> backend.remove_file('/f.ext', '1.0.0') - >>> backend.exists('/f.ext', '1.0.0') - False """ - path_with_version = self._path_with_version(path, version) - + path = utils.check_path(path) utils.call_function_on_backend( self._remove_file, - path_with_version, + path, ) @property def sep(self) -> str: - r"""File separator on backend.""" + r"""File separator on backend. + + Returns: file separator + + """ return utils.BACKEND_SEPARATOR def split( @@ -937,16 +652,6 @@ def split( ValueError: if ``path`` does not start with ``'/'`` or does not match ``'[A-Za-z0-9/._-]+'`` - Examples: - >>> backend.split('/') - ('/', '') - >>> backend.split('/f.ext') - ('/', 'f.ext') - >>> backend.split('/sub/') - ('/sub/', '') - >>> backend.split('/sub//f.ext') - ('/sub/', 'f.ext') - """ path = utils.check_path(path) @@ -954,88 +659,3 @@ def split( basename = path.split(self.sep)[-1] return root, basename - - def versions( - self, - path: str, - *, - suppress_backend_errors: bool = False, - ) -> typing.List[str]: - r"""Versions of a file. - - Args: - path: path to file on backend - suppress_backend_errors: if set to ``True``, - silently catch errors raised on the backend - and return an empty list - - Returns: - list of versions in ascending order - - Raises: - BackendError: if ``suppress_backend_errors`` is ``False`` - and an error is raised on the backend, - e.g. ``path`` does not exist - ValueError: if ``path`` does not start with ``'/'`` or - does not match ``'[A-Za-z0-9/._-]+'`` - - Examples: - >>> backend.versions('/f.ext') - ['1.0.0', '2.0.0'] - - """ - paths = self.ls(path, suppress_backend_errors=suppress_backend_errors) - vs = [v for _, v in paths] - return vs - - def _use_legacy_file_structure( - self, - *, - extensions: typing.List[str] = None, - regex: bool = False, - ): - r"""Use legacy file structure. - - Stores files under - ``'...///-.'`` - instead of - ``'...//'``. - By default, - the extension - ```` - is set to the string after the last dot. - I.e., - the backend path - ``'.../file.tar.gz'`` - will translate into - ``'.../file.tar/1.0.0/file.tar-1.0.0.gz'``. - However, - by passing a list with custom extensions - it is possible to overwrite - the default behavior - for certain extensions. - E.g., - with - ``backend._use_legacy_file_structure(extensions=['tar.gz'])`` - it is ensured that - ``'tar.gz'`` - will be recognized as an extension - and the backend path - ``'.../file.tar.gz'`` - will then translate into - ``'.../file/1.0.0/file-1.0.0.tar.gz'``. - If ``regex`` is set to ``True``, - the extensions are treated as regular expressions. - E.g. - with - ``backend._use_legacy_file_structure(extensions=['\d+.tar.gz'], - regex=True)`` - the backend path - ``'.../file.99.tar.gz'`` - will translate into - ``'.../file/1.0.0/file-1.0.0.99.tar.gz'``. - - """ - self._legacy_file_structure = True - self._legacy_extensions = extensions or [] - self._legacy_file_structure_regex = regex diff --git a/audbackend/core/conftest.py b/audbackend/core/conftest.py index 4492f46d..6992159d 100644 --- a/audbackend/core/conftest.py +++ b/audbackend/core/conftest.py @@ -38,24 +38,52 @@ def prepare_docstring_tests(doctest_namespace): current_dir = os.getcwd() os.chdir(tmp) - host = 'host' - repository = 'doctest' + file = 'src.pth' + audeer.touch(file) audbackend.register('file-system', DoctestFileSystem) - backend = audbackend.create('file-system', host, repository) - file = 'src.pth' - audeer.touch(file) - backend.put_archive('.', '/a.zip', '1.0.0', files=[file]) - backend.put_file(file, '/a/b.ext', '1.0.0') - for version in ['1.0.0', '2.0.0']: - backend.put_file(file, '/f.ext', version) + # backend + backend = audbackend.Backend('host', 'repo') doctest_namespace['backend'] = backend + interface = audbackend.interface.Base(backend) + doctest_namespace['interface'] = interface + + # versioned interface + + versioned = audbackend.create( + 'file-system', + 'host', + 'repo', + interface=audbackend.interface.Versioned, + ) + assert isinstance(versioned, audbackend.interface.Versioned) + versioned.put_archive('.', '/a.zip', '1.0.0', files=[file]) + versioned.put_file(file, '/a/b.ext', '1.0.0') + for version in ['1.0.0', '2.0.0']: + versioned.put_file(file, '/f.ext', version) + doctest_namespace['versioned'] = versioned + + # unversioned interface + + unversioned = audbackend.create( + 'file-system', + 'host', + 'repo-unversioned', + interface=audbackend.interface.Unversioned, + ) + assert isinstance(unversioned, audbackend.interface.Unversioned) + unversioned.put_archive('.', '/a.zip', files=[file]) + unversioned.put_file(file, '/a/b.ext') + unversioned.put_file(file, '/f.ext') + doctest_namespace['unversioned'] = unversioned + yield - audbackend.delete('file-system', host, repository) + audbackend.delete('file-system', 'host', 'repo') + audbackend.delete('file-system', 'host', 'repo-unversioned') audbackend.register('file-system', audbackend.FileSystem) os.chdir(current_dir) diff --git a/audbackend/core/errors.py b/audbackend/core/errors.py index 8430c223..26a167c9 100644 --- a/audbackend/core/errors.py +++ b/audbackend/core/errors.py @@ -6,7 +6,12 @@ class BackendError(Exception): Examples: >>> try: - ... backend.checksum('/does/not/exist', '1.0.0') + ... unversioned.checksum('/does/not/exist') + ... except BackendError as ex: + ... ex.exception + FileNotFoundError(2, 'No such file or directory') + >>> try: + ... versioned.checksum('/does/not/exist', '1.0.0') ... except BackendError as ex: ... ex.exception FileNotFoundError(2, 'No such file or directory') diff --git a/audbackend/core/interface/base.py b/audbackend/core/interface/base.py new file mode 100644 index 00000000..657cf90f --- /dev/null +++ b/audbackend/core/interface/base.py @@ -0,0 +1,144 @@ +import typing + +from audbackend.core.backend import Backend + + +class Base: + r"""Interface base class. + + Provides an interface to a backend, + see e.g. + :class:`audbackend.Unversioned` + and + :class:`audbackend.Versioned`. + + Derive from this class to + create a new interface. + + Args: + backend: backend object + + """ + def __init__( + self, + backend: Backend, + ): + self._backend = backend + + def __repr__(self) -> str: # noqa: D105 + name = f'{self.__class__.__module__}.{self.__class__.__name__}' + return f'{name}{self.backend}' + + @property + def backend(self) -> Backend: + r"""Backend object. + + Returns: + backend object + + Examples: + >>> interface.backend + ('audbackend.core.backend.Backend', 'host', 'repo') + + """ + return self._backend + + @property + def host(self) -> str: + r"""Host path. + + Returns: host path + + Examples: + >>> interface.host + 'host' + + """ + return self.backend.host + + def join( + self, + path: str, + *paths, + ) -> str: + r"""Join to path on backend. + + Args: + path: first part of path + *paths: additional parts of path + + Returns: + path joined by :attr:`Backend.sep` + + Raises: + ValueError: if ``path`` contains invalid character + or does not start with ``'/'``, + or if joined path contains invalid character + + Examples: + >>> interface.join('/', 'f.ext') + '/f.ext' + >>> interface.join('/sub', 'f.ext') + '/sub/f.ext' + >>> interface.join('//sub//', '/', '', None, '/f.ext') + '/sub/f.ext' + + """ + return self.backend.join(path, *paths) + + @property + def repository(self) -> str: + r"""Repository name. + + Returns: + repository name + + Examples: + >>> interface.repository + 'repo' + + """ + return self.backend.repository + + @property + def sep(self) -> str: + r"""File separator on backend. + + Returns: + file separator + + Examples: + >>> interface.sep + '/' + + """ + return self.backend.sep + + def split( + self, + path: str, + ) -> typing.Tuple[str, str]: + r"""Split path on backend into sub-path and basename. + + Args: + path: path containing :attr:`Backend.sep` as separator + + Returns: + tuple containing (root, basename) + + Raises: + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> interface.split('/') + ('/', '') + >>> interface.split('/f.ext') + ('/', 'f.ext') + >>> interface.split('/sub/') + ('/sub/', '') + >>> interface.split('/sub//f.ext') + ('/sub/', 'f.ext') + + """ + return self.backend.split(path) diff --git a/audbackend/core/interface/unversioned.py b/audbackend/core/interface/unversioned.py new file mode 100644 index 00000000..0fadc62c --- /dev/null +++ b/audbackend/core/interface/unversioned.py @@ -0,0 +1,417 @@ +import os # noqa: F401 +import typing + +from audbackend.core.interface.base import Base + + +class Unversioned(Base): + r"""Interface for unversioned file access. + + Use this interface if you don't care about versioning. + For every backend path exactly one file exists on the backend. + + """ + + def checksum( + self, + path: str, + ) -> str: + r"""MD5 checksum for file on backend. + + Args: + path: path to file on backend + + Returns: + MD5 checksum + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.checksum('/f.ext') + 'd41d8cd98f00b204e9800998ecf8427e' + + """ + return self.backend.checksum(path) + + def date( + self, + path: str, + ) -> str: + r"""Last modification date of file on backend. + + If the date cannot be determined, + an empty string is returned. + + Args: + path: path to file on backend + + Returns: + date in format ``'yyyy-mm-dd'`` + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.date('/f.ext') + '1991-02-20' + + """ + return self.backend.date(path) + + def exists( + self, + path: str, + *, + suppress_backend_errors: bool = False, + ) -> bool: + r"""Check if file exists on backend. + + Args: + path: path to file on backend + suppress_backend_errors: if set to ``True``, + silently catch errors raised on the backend + and return ``False`` + + Returns: + ``True`` if file exists + + Raises: + BackendError: if ``suppress_backend_errors`` is ``False`` + and an error is raised on the backend, + e.g. due to a connection timeout + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> unversioned.exists('/f.ext') + True + + """ + return self.backend.exists( + path, + suppress_backend_errors=suppress_backend_errors, + ) + + def get_archive( + self, + src_path: str, + dst_root: str, + *, + tmp_root: str = None, + verbose: bool = False, + ) -> typing.List[str]: + r"""Get archive from backend and extract. + + The archive type is derived from the extension of ``src_path``. + See :func:`audeer.extract_archive` for supported extensions. + + If ``dst_root`` does not exist, + it is created. + + Args: + src_path: path to archive on backend + dst_root: local destination directory + tmp_root: directory under which archive is temporarily extracted. + Defaults to temporary directory of system + verbose: show debug messages + + Returns: + extracted files + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``src_path`` does not exist + FileNotFoundError: if ``tmp_root`` does not exist + NotADirectoryError: if ``dst_root`` is not a directory + PermissionError: if the user lacks write permissions + for ``dst_path`` + RuntimeError: if extension of ``src_path`` is not supported + or ``src_path`` is a malformed archive + ValueError: if ``src_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.get_archive('/a.zip', '.') + ['src.pth'] + + """ + return self.backend.get_archive( + src_path, + dst_root, + tmp_root=tmp_root, + verbose=verbose, + ) + + def get_file( + self, + src_path: str, + dst_path: str, + *, + verbose: bool = False, + ) -> str: + r"""Get file from backend. + + If the folder of + ``dst_path`` does not exist, + it is created. + + If ``dst_path`` exists + with a different checksum, + it is overwritten, + or otherwise, + the operation is silently skipped. + + To ensure the file is completely retrieved, + it is first stored in a temporary directory + and afterwards moved to ``dst_path``. + + Args: + src_path: path to file on backend + dst_path: destination path to local file + verbose: show debug messages + + Returns: + full path to local file + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``src_path`` does not exist + IsADirectoryError: if ``dst_path`` points to an existing folder + PermissionError: if the user lacks write permissions + for ``dst_path`` + ValueError: if ``src_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> os.path.exists('dst.pth') + False + >>> _ = unversioned.get_file('/f.ext', 'dst.pth') + >>> os.path.exists('dst.pth') + True + + """ + return self.backend.get_file(src_path, dst_path, verbose=verbose) + + def ls( + self, + path: str = '/', + *, + pattern: str = None, + suppress_backend_errors: bool = False, + ) -> typing.List[str]: + r"""List files on backend. + + Returns a sorted list of tuples + with path and version. + If a full path + (e.g. ``/sub/file.ext``) + is provided, + all versions of the path are returned. + If a sub-path + (e.g. ``/sub/``) + is provided, + all files that start with + the sub-path are returned. + When ``path`` is set to ``'/'`` + a (possibly empty) list with + all files on the backend is returned. + + Args: + path: path or sub-path + (if it ends with ``'/'``) + on backend + pattern: if not ``None``, + return only files matching the pattern string, + see :func:`fnmatch.fnmatch` + suppress_backend_errors: if set to ``True``, + silently catch errors raised on the backend + and return an empty list + + Returns: + list of tuples (path, version) + + Raises: + BackendError: if ``suppress_backend_errors`` is ``False`` + and an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.ls() + ['/a.zip', '/a/b.ext', '/f.ext'] + >>> unversioned.ls('/f.ext') + ['/f.ext'] + >>> unversioned.ls(pattern='*.ext') + ['/a/b.ext', '/f.ext'] + >>> unversioned.ls(pattern='b.*') + ['/a/b.ext'] + >>> unversioned.ls('/a/') + ['/a/b.ext'] + + """ # noqa: E501 + return self.backend.ls( + path, + pattern=pattern, + suppress_backend_errors=suppress_backend_errors, + ) + + def owner( + self, + path: str, + ) -> str: + r"""Owner of file on backend. + + If the owner of the file + cannot be determined, + an empty string is returned. + + Args: + path: path to file on backend + + Returns: + owner + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.owner('/f.ext') + 'doctest' + + """ + return self.backend.owner(path) + + def put_archive( + self, + src_root: str, + dst_path: str, + *, + files: typing.Union[str, typing.Sequence[str]] = None, + tmp_root: str = None, + verbose: bool = False, + ): + r"""Create archive and put on backend. + + The archive type is derived from the extension of ``dst_path``. + See :func:`audeer.create_archive` for supported extensions. + + The operation is silently skipped, + if an archive with the same checksum + already exists on the backend. + + Args: + src_root: local root directory where files are located. + By default, + all files below ``src_root`` + will be included into the archive. + Use ``files`` to select specific files + dst_path: path to archive on backend + files: file(s) to include into the archive. + Must exist within ``src_root`` + tmp_root: directory under which archive is temporarily created. + Defaults to temporary directory of system + verbose: show debug messages + + Raises: + BackendError: if an error is raised on the backend + FileNotFoundError: if ``src_root``, + ``tmp_root``, + or one or more ``files`` do not exist + NotADirectoryError: if ``src_root`` is not a folder + RuntimeError: if ``dst_path`` does not end with + ``zip`` or ``tar.gz`` + or a file in ``files`` is not below ``root`` + ValueError: if ``dst_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.exists('/a.tar.gz') + False + >>> unversioned.put_archive('.', '/a.tar.gz') + >>> unversioned.exists('/a.tar.gz') + True + + """ + self.backend.put_archive( + src_root, + dst_path, + files=files, + tmp_root=tmp_root, + verbose=verbose, + ) + + def put_file( + self, + src_path: str, + dst_path: str, + *, + verbose: bool = False, + ): + r"""Put file on backend. + + The operation is silently skipped, + if a file with the same checksum + already exists on the backend. + + Args: + src_path: path to local file + dst_path: path to file on backend + verbose: show debug messages + + Returns: + file path on backend + + Raises: + BackendError: if an error is raised on the backend + FileNotFoundError: if ``src_path`` does not exist + IsADirectoryError: if ``src_path`` is a folder + ValueError: if ``dst_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.exists('/sub/f.ext') + False + >>> unversioned.put_file('src.pth', '/sub/f.ext') + >>> unversioned.exists('/sub/f.ext') + True + + """ + self.backend.put_file(src_path, dst_path, verbose=verbose) + + def remove_file( + self, + path: str, + ): + r"""Remove file from backend. + + Args: + path: path to file on backend + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> unversioned.exists('/f.ext') + True + >>> unversioned.remove_file('/f.ext') + >>> unversioned.exists('/f.ext') + False + + """ + self.backend.remove_file(path) diff --git a/audbackend/core/interface/versioned.py b/audbackend/core/interface/versioned.py new file mode 100644 index 00000000..c10b5747 --- /dev/null +++ b/audbackend/core/interface/versioned.py @@ -0,0 +1,746 @@ +import os +import re +import typing + +import audeer + +from audbackend.core import utils +from audbackend.core.backend import Backend +from audbackend.core.errors import BackendError +from audbackend.core.interface.base import Base + + +class Versioned(Base): + r"""Interface for versioned file access. + + Use this interface if you care about versioning. + For each file on the backend path one or more versions may exist. + + """ + + def __init__( + self, + backend: Backend, + ): + super().__init__(backend) + + # to support legacy file structure + # see _use_legacy_file_structure() + self._legacy_extensions = [] + self._legacy_file_structure = False + self._legacy_file_structure_regex = False + + def checksum( + self, + path: str, + version: str, + ) -> str: + r"""MD5 checksum for file on backend. + + Args: + path: path to file on backend + version: version string + + Returns: + MD5 checksum + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.checksum('/f.ext', '1.0.0') + 'd41d8cd98f00b204e9800998ecf8427e' + + """ + path_with_version = self._path_with_version(path, version) + return self.backend.checksum(path_with_version) + + def date( + self, + path: str, + version: str, + ) -> str: + r"""Last modification date of file on backend. + + If the date cannot be determined, + an empty string is returned. + + Args: + path: path to file on backend + version: version string + + Returns: + date in format ``'yyyy-mm-dd'`` + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.date('/f.ext', '1.0.0') + '1991-02-20' + + """ + path_with_version = self._path_with_version(path, version) + return self.backend.date(path_with_version) + + def exists( + self, + path: str, + version: str, + *, + suppress_backend_errors: bool = False, + ) -> bool: + r"""Check if file exists on backend. + + Args: + path: path to file on backend + version: version string + suppress_backend_errors: if set to ``True``, + silently catch errors raised on the backend + and return ``False`` + + Returns: + ``True`` if file exists + + Raises: + BackendError: if ``suppress_backend_errors`` is ``False`` + and an error is raised on the backend, + e.g. due to a connection timeout + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.exists('/f.ext', '1.0.0') + True + + """ + path_with_version = self._path_with_version(path, version) + return self.backend.exists( + path_with_version, + suppress_backend_errors=suppress_backend_errors, + ) + + def get_archive( + self, + src_path: str, + dst_root: str, + version: str, + *, + tmp_root: str = None, + verbose: bool = False, + ) -> typing.List[str]: + r"""Get archive from backend and extract. + + The archive type is derived from the extension of ``src_path``. + See :func:`audeer.extract_archive` for supported extensions. + + If ``dst_root`` does not exist, + it is created. + + Args: + src_path: path to archive on backend + dst_root: local destination directory + version: version string + tmp_root: directory under which archive is temporarily extracted. + Defaults to temporary directory of system + verbose: show debug messages + + Returns: + extracted files + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``src_path`` does not exist + FileNotFoundError: if ``tmp_root`` does not exist + NotADirectoryError: if ``dst_root`` is not a directory + PermissionError: if the user lacks write permissions + for ``dst_path`` + RuntimeError: if extension of ``src_path`` is not supported + or ``src_path`` is a malformed archive + ValueError: if ``src_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.get_archive('/a.zip', '.', '1.0.0') + ['src.pth'] + + """ + src_path_with_version = self._path_with_version(src_path, version) + return self.backend.get_archive( + src_path_with_version, + dst_root, + tmp_root=tmp_root, + verbose=verbose, + ) + + def get_file( + self, + src_path: str, + dst_path: str, + version: str, + *, + verbose: bool = False, + ) -> str: + r"""Get file from backend. + + If the folder of + ``dst_path`` does not exist, + it is created. + + If ``dst_path`` exists + with a different checksum, + it is overwritten, + or otherwise, + the operation is silently skipped. + + To ensure the file is completely retrieved, + it is first stored in a temporary directory + and afterwards moved to ``dst_path``. + + Args: + src_path: path to file on backend + dst_path: destination path to local file + version: version string + verbose: show debug messages + + Returns: + full path to local file + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``src_path`` does not exist + IsADirectoryError: if ``dst_path`` points to an existing folder + PermissionError: if the user lacks write permissions + for ``dst_path`` + ValueError: if ``src_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> os.path.exists('dst.pth') + False + >>> _ = versioned.get_file('/f.ext', 'dst.pth', '1.0.0') + >>> os.path.exists('dst.pth') + True + + """ + src_path_with_version = self._path_with_version(src_path, version) + return self.backend.get_file( + src_path_with_version, + dst_path, + verbose=verbose, + ) + + def latest_version( + self, + path: str, + ) -> str: + r"""Latest version of a file. + + Args: + path: path to file on backend + + Returns: + version string + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> versioned.latest_version('/f.ext') + '2.0.0' + + """ + vs = self.versions(path) + return vs[-1] + + def ls( + self, + path: str = '/', + *, + latest_version: bool = False, + pattern: str = None, + suppress_backend_errors: bool = False, + ) -> typing.List[typing.Tuple[str, str]]: + r"""List files on backend. + + Returns a sorted list of tuples + with path and version. + If a full path + (e.g. ``/sub/file.ext``) + is provided, + all versions of the path are returned. + If a sub-path + (e.g. ``/sub/``) + is provided, + all files that start with + the sub-path are returned. + When ``path`` is set to ``'/'`` + a (possibly empty) list with + all files on the backend is returned. + + Args: + path: path or sub-path + (if it ends with ``'/'``) + on backend + latest_version: if multiple versions of a file exist, + only include the latest + pattern: if not ``None``, + return only files matching the pattern string, + see :func:`fnmatch.fnmatch` + suppress_backend_errors: if set to ``True``, + silently catch errors raised on the backend + and return an empty list + + Returns: + list of tuples (path, version) + + Raises: + BackendError: if ``suppress_backend_errors`` is ``False`` + and an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> versioned.ls() + [('/a.zip', '1.0.0'), ('/a/b.ext', '1.0.0'), ('/f.ext', '1.0.0'), ('/f.ext', '2.0.0')] + >>> versioned.ls(latest_version=True) + [('/a.zip', '1.0.0'), ('/a/b.ext', '1.0.0'), ('/f.ext', '2.0.0')] + >>> versioned.ls('/f.ext') + [('/f.ext', '1.0.0'), ('/f.ext', '2.0.0')] + >>> versioned.ls(pattern='*.ext') + [('/a/b.ext', '1.0.0'), ('/f.ext', '1.0.0'), ('/f.ext', '2.0.0')] + >>> versioned.ls(pattern='b.*') + [('/a/b.ext', '1.0.0')] + >>> versioned.ls('/a/') + [('/a/b.ext', '1.0.0')] + + """ # noqa: E501 + if path.endswith('/'): # find files under sub-path + + paths = self.backend.ls( + path, + pattern=pattern, + suppress_backend_errors=suppress_backend_errors, + ) + + else: # find versions of path + + root, file = self.split(path) + + paths = self.backend.ls( + root, + pattern=pattern, + suppress_backend_errors=suppress_backend_errors, + ) + + # filter for '/root/version/file' + if self._legacy_file_structure: + depth = root.count('/') + 2 + name, ext = self._legacy_split_ext(file) + match = re.compile(rf'{name}-\d+\.\d+.\d+{ext}') + paths = [ + p for p in paths + if ( + p.count('/') == depth and + match.match(os.path.basename(p)) + ) + ] + else: + depth = root.count('/') + 1 + paths = [ + p for p in paths + if ( + p.count('/') == depth and + os.path.basename(p) == file + ) + ] + + if not paths and not suppress_backend_errors: + # since the backend does no longer raise an error + # if the path does not exist + # we have to do it + try: + utils.raise_file_not_found_error(path) + except FileNotFoundError as ex: + raise BackendError(ex) + + if not paths: + return [] + + paths_and_versions = [] + for p in paths: + + tokens = p.split(self.sep) + + name = tokens[-1] + version = tokens[-2] + + if version: + + if self._legacy_file_structure: + base = tokens[-3] + ext = name[len(base) + len(version) + 1:] + name = f'{base}{ext}' + path = self.sep.join(tokens[:-3]) + else: + path = self.sep.join(tokens[:-2]) + + path = self.sep + path + path = self.join(path, name) + + paths_and_versions.append((path, version)) + + paths_and_versions = sorted(paths_and_versions) + + if latest_version: + # d[path] = ['1.0.0', '2.0.0'] + d = {} + for p, v in paths_and_versions: + if p not in d: + d[p] = [] + d[p].append(v) + # d[path] = '2.0.0' + for p, vs in d.items(): + d[p] = audeer.sort_versions(vs)[-1] + # [(path, '2.0.0')] + paths_and_versions = [(p, v) for p, v in d.items()] + + return paths_and_versions + + def owner( + self, + path: str, + version: str, + ) -> str: + r"""Owner of file on backend. + + If the owner of the file + cannot be determined, + an empty string is returned. + + Args: + path: path to file on backend + version: version string + + Returns: + owner + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.owner('/f.ext', '1.0.0') + 'doctest' + + """ + path_with_version = self._path_with_version(path, version) + return self.backend.owner(path_with_version) + + def put_archive( + self, + src_root: str, + dst_path: str, + version: str, + *, + files: typing.Union[str, typing.Sequence[str]] = None, + tmp_root: str = None, + verbose: bool = False, + ): + r"""Create archive and put on backend. + + The archive type is derived from the extension of ``dst_path``. + See :func:`audeer.create_archive` for supported extensions. + + The operation is silently skipped, + if an archive with the same checksum + already exists on the backend. + + Args: + src_root: local root directory where files are located. + By default, + all files below ``src_root`` + will be included into the archive. + Use ``files`` to select specific files + dst_path: path to archive on backend + version: version string + files: file(s) to include into the archive. + Must exist within ``src_root`` + tmp_root: directory under which archive is temporarily created. + Defaults to temporary directory of system + verbose: show debug messages + + Raises: + BackendError: if an error is raised on the backend + FileNotFoundError: if ``src_root``, + ``tmp_root``, + or one or more ``files`` do not exist + NotADirectoryError: if ``src_root`` is not a folder + RuntimeError: if ``dst_path`` does not end with + ``zip`` or ``tar.gz`` + or a file in ``files`` is not below ``root`` + ValueError: if ``dst_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.exists('/a.tar.gz', '1.0.0') + False + >>> versioned.put_archive('.', '/a.tar.gz', '1.0.0') + >>> versioned.exists('/a.tar.gz', '1.0.0') + True + + """ + dst_path_with_version = self._path_with_version(dst_path, version) + self.backend.put_archive( + src_root, + dst_path_with_version, + files=files, + tmp_root=tmp_root, + verbose=verbose, + ) + + def put_file( + self, + src_path: str, + dst_path: str, + version: str, + *, + verbose: bool = False, + ): + r"""Put file on backend. + + The operation is silently skipped, + if a file with the same checksum + already exists on the backend. + + Args: + src_path: path to local file + dst_path: path to file on backend + version: version string + verbose: show debug messages + + Returns: + file path on backend + + Raises: + BackendError: if an error is raised on the backend + FileNotFoundError: if ``src_path`` does not exist + IsADirectoryError: if ``src_path`` is a folder + ValueError: if ``dst_path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.exists('/sub/f.ext', '3.0.0') + False + >>> versioned.put_file('src.pth', '/sub/f.ext', '3.0.0') + >>> versioned.exists('/sub/f.ext', '3.0.0') + True + + """ + dst_path_with_version = self._path_with_version(dst_path, version) + return self.backend.put_file( + src_path, + dst_path_with_version, + verbose=verbose, + ) + + def remove_file( + self, + path: str, + version: str, + ): + r"""Remove file from backend. + + Args: + path: path to file on backend + version: version string + + Raises: + BackendError: if an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + ValueError: if ``version`` is empty or + does not match ``'[A-Za-z0-9._-]+'`` + + Examples: + >>> versioned.exists('/f.ext', '1.0.0') + True + >>> versioned.remove_file('/f.ext', '1.0.0') + >>> versioned.exists('/f.ext', '1.0.0') + False + + """ + path_with_version = self._path_with_version(path, version) + self.backend.remove_file(path_with_version) + + def versions( + self, + path: str, + *, + suppress_backend_errors: bool = False, + ) -> typing.List[str]: + r"""Versions of a file. + + Args: + path: path to file on backend + suppress_backend_errors: if set to ``True``, + silently catch errors raised on the backend + and return an empty list + + Returns: + list of versions in ascending order + + Raises: + BackendError: if ``suppress_backend_errors`` is ``False`` + and an error is raised on the backend, + e.g. ``path`` does not exist + ValueError: if ``path`` does not start with ``'/'`` or + does not match ``'[A-Za-z0-9/._-]+'`` + + Examples: + >>> versioned.versions('/f.ext') + ['1.0.0', '2.0.0'] + + """ + paths = self.ls(path, suppress_backend_errors=suppress_backend_errors) + vs = [v for _, v in paths] + return vs + + def _legacy_split_ext( + self, + name: str, + ) -> typing.Tuple[str, str]: + r"""Split name into basename and extension.""" + ext = None + for custom_ext in self._legacy_extensions: + # check for custom extension + # ensure basename is not empty + if self._legacy_file_structure_regex: + pattern = rf'\.({custom_ext})$' + match = re.search(pattern, name[1:]) + if match: + ext = match.group(1) + elif name[1:].endswith(f'.{custom_ext}'): + ext = custom_ext + if ext is None: + # if no custom extension is found + # use last string after dot + ext = audeer.file_extension(name) + + base = audeer.replace_file_extension(name, '', ext=ext) + + if ext: + ext = f'.{ext}' + + return base, ext + + def _path_with_version( + self, + path: str, + version: str, + ) -> str: + r"""Convert to versioned path. + + / + -> + // + + or legacy: + + / + -> + ///- + + """ + version = utils.check_version(version) + + root, name = self.split(path) + + if self._legacy_file_structure: + base, ext = self._legacy_split_ext(name) + path = self.join(root, base, version, f'{base}-{version}{ext}') + else: + path = self.join(root, version, name) + + return path + + def _use_legacy_file_structure( + self, + *, + extensions: typing.List[str] = None, + regex: bool = False, + ): + r"""Use legacy file structure. + + Stores files under + ``'...///-.'`` + instead of + ``'...//'``. + By default, + the extension + ```` + is set to the string after the last dot. + I.e., + the backend path + ``'.../file.tar.gz'`` + will translate into + ``'.../file.tar/1.0.0/file.tar-1.0.0.gz'``. + However, + by passing a list with custom extensions + it is possible to overwrite + the default behavior + for certain extensions. + E.g., + with + ``backend._use_legacy_file_structure(extensions=['tar.gz'])`` + it is ensured that + ``'tar.gz'`` + will be recognized as an extension + and the backend path + ``'.../file.tar.gz'`` + will then translate into + ``'.../file/1.0.0/file-1.0.0.tar.gz'``. + If ``regex`` is set to ``True``, + the extensions are treated as regular expressions. + E.g. + with + ``backend._use_legacy_file_structure(extensions=['\d+.tar.gz'], + regex=True)`` + the backend path + ``'.../file.99.tar.gz'`` + will translate into + ``'.../file/1.0.0/file-1.0.0.99.tar.gz'``. + + """ + self._legacy_file_structure = True + self._legacy_extensions = extensions or [] + self._legacy_file_structure_regex = regex diff --git a/audbackend/interface/__init__.py b/audbackend/interface/__init__.py new file mode 100644 index 00000000..4770e576 --- /dev/null +++ b/audbackend/interface/__init__.py @@ -0,0 +1,3 @@ +from audbackend.core.interface.base import Base +from audbackend.core.interface.unversioned import Unversioned +from audbackend.core.interface.versioned import Versioned diff --git a/docs/api-src/audbackend.interface.rst b/docs/api-src/audbackend.interface.rst new file mode 100644 index 00000000..b59e74ed --- /dev/null +++ b/docs/api-src/audbackend.interface.rst @@ -0,0 +1,27 @@ +audbackend.interface +==================== + +.. automodule:: audbackend.interface + +To access the files on a backend users +can choose between two interfaces +:class:`audbackend.interface.Unversioned` +or +:class:`audbackend.interface.Versioned`: + +.. autosummary:: + :toctree: + :nosignatures: + + Unversioned + Versioned + +Users can implement their own +interface by deriving from +:class:`audbackend.interface.Base` + +.. autosummary:: + :toctree: + :nosignatures: + + Base diff --git a/docs/index.rst b/docs/index.rst index 255bd427..9070cdaf 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ :hidden: api/audbackend + api/audbackend.interface genindex .. toctree:: diff --git a/docs/usage.rst b/docs/usage.rst index 185b833a..8f85dda5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -11,11 +11,14 @@ that have not been invented yet :) This tutorial is divided -into two parts. +into three parts. Under :ref:`file-system-example`, we show the basic usage by means of a standard file system. +In :ref:`backend-interface` +we introduce the concept +of a backend interface. Under :ref:`develop-new-backend`, we take a deep dive and develop a backend @@ -73,8 +76,9 @@ we use :func:`audbackend.create` to instantiate a backend instead of calling the class ourselves. When creating the instance -we provide two arguments: +we provide three arguments: +* ``name``: the name under which the backend class is registered * ``host``: the host address, in this case a folder on the local file system. * ``repository``: the repository name, @@ -279,6 +283,128 @@ exception thrown by the backend. display(str(ex.exception)) +.. _backend-interface: + +Backend interface +----------------- + +By default, +a file is stored +on the backend with a version +(see :ref:`file-system-example`). +We can change this behavior +by using a different interface. +For instance, +if we are not interested +in versioning we can use +:class:`audbackend.interface.Unversioned`. + +.. jupyter-execute:: + + backend = audbackend.create( + 'file-system', + './host', + 'repo', + interface=audbackend.interface.Unversioned, + ) + backend.put_file('local.txt', '/file-without-version.txt') + backend.ls() + +.. jupyter-execute:: + :hide-code: + :hide-output: + + audbackend.delete('file-system', './host', 'repo') + + +We can also implement our own interface +by deriving from +:class:`audbackend.interface.Base`. +For instance, +we can create an interface +to manage user content. +It provides three functions: + +* ``add_user()`` to register a user +* ``upload()`` to upload a file for user +* ``ls()`` to list the files of a user + +We store user information +in a database under +``'/user.map'``. +To access and update +the database +we implement the following +helper class. + + +.. jupyter-execute:: + + import shelve + + class UserDB: + r"""User database. + + Temporarily get user database + and write changes back to the backend. + + """ + def __init__(self, backend: audbackend.Backend): + self.backend = backend + + def __enter__(self) -> shelve.Shelf: + if self.backend.exists('/user.db'): + self.backend.get_file('/user.db', '~.db') + self._map = shelve.open('~.db', flag='w', writeback=True) + else: + self._map = shelve.open('~.db', writeback=True) + return self._map + + def __exit__(self, exc_type, exc_val, exc_tb): + self._map.close() + self.backend.put_file('~.db', '/user.db') + os.remove('~.db') + + +Now, +we implement the interface. + +.. jupyter-execute:: + + class UserContent(audbackend.interface.Base): + + def add_user(self, username: str, password: str): + r"""Add user to database.""" + with UserDB(self.backend) as map: + map[username] = password + + def upload(self, username: str, password: str, path: str): + r"""Upload user file.""" + with UserDB(self.backend) as map: + if username not in map or map[username] != password: + raise ValueError('User does not exist or wrong password.') + self.backend.put_file(path, f'/{username}/{os.path.basename(path)}') + + def ls(self, username: str) -> list: + r"""List files of user.""" + with UserDB(self.backend) as map: + if username not in map: + return [] + return self.backend.ls(f'/{username}/') + + +Let's create a backend +with our custom interface: + +.. jupyter-execute:: + + backend = audbackend.create('file-system', tmp, 'repo', interface=UserContent) + + backend.add_user('audeering', 'pa$$word') + backend.upload('audeering', 'pa$$word', 'local.txt') + backend.ls('audeering') + + .. _develop-new-backend: Develop new backend diff --git a/pyproject.toml b/pyproject.toml index 515f6bea..614b9f1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ addopts = ''' --cov-report term-missing --cov-report xml --ignore=tests/misc/ - --ignore=tests/test_filesystem_only.py + --ignore=tests/test_backend_filesystem_only.py ''' diff --git a/tests/conftest.py b/tests/conftest.py index 75466224..69c9d398 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,6 +18,15 @@ if os.name != 'nt': pytest.BACKENDS.append('single-folder') + +pytest.UNVERSIONED = [ + (backend, audbackend.interface.Unversioned) + for backend in pytest.BACKENDS +] +pytest.VERSIONED = [ + ('file-system', audbackend.interface.Versioned) +] + # UID for test session # Repositories on the host will be named # unittest-- @@ -57,13 +66,13 @@ def owner(request): @pytest.fixture(scope='function', autouse=False) -def backend(hosts, request): +def interface(hosts, request): - name = request.param + name, interface = request.param host = hosts[name] repository = f'unittest-{pytest.UID}-{audeer.uid()[:8]}' - backend = audbackend.create(name, host, repository) + backend = audbackend.create(name, host, repository, interface=interface) yield backend diff --git a/tests/test_api.py b/tests/test_api.py index 49d00f9d..986d3edd 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -63,19 +63,19 @@ def test_api(hosts, name, host, repository, cls): with pytest.raises(audbackend.BackendError, match=error_msg): audbackend.access(name, host, repository) - backend = audbackend.create(name, host, repository) - assert isinstance(backend, cls) + interface = audbackend.create(name, host, repository) + assert isinstance(interface.backend, cls) with pytest.raises(audbackend.BackendError, match=error_msg): audbackend.create(name, host, repository) - backend = audbackend.access(name, host, repository) - assert isinstance(backend, cls) - assert backend in audbackend.available()[name] + interface = audbackend.access(name, host, repository) + assert isinstance(interface.backend, cls) + assert interface.backend in audbackend.available()[name] audbackend.delete(name, host, repository) - assert backend not in audbackend.available()[name] + assert interface.backend not in audbackend.available()[name] with pytest.raises(audbackend.BackendError, match=error_msg): audbackend.access(name, host, repository) diff --git a/tests/test_artifactory.py b/tests/test_backend_artifactory.py similarity index 81% rename from tests/test_artifactory.py rename to tests/test_backend_artifactory.py index 367ae8e9..863adb7e 100644 --- a/tests/test_artifactory.py +++ b/tests/test_backend_artifactory.py @@ -75,17 +75,17 @@ def test_authentication(tmpdir, hosts, hide_credentials): @pytest.mark.parametrize( - 'backend', - ['artifactory'], + 'interface', + [('artifactory', audbackend.interface.Versioned)], indirect=True, ) -def test_errors(tmpdir, backend): +def test_errors(tmpdir, interface): - backend._username = 'non-existing' - backend._api_key = 'non-existing' + interface.backend._username = 'non-existing' + interface.backend._api_key = 'non-existing' local_file = audeer.touch(audeer.path(tmpdir, 'file.txt')) - remote_file = backend.join( + remote_file = interface.join( '/', audeer.uid()[:8], 'file.txt', @@ -94,8 +94,8 @@ def test_errors(tmpdir, backend): # --- exists --- with pytest.raises(audbackend.BackendError): - backend.exists(remote_file, version) - assert backend.exists( + interface.exists(remote_file, version) + assert interface.exists( remote_file, version, suppress_backend_errors=True, @@ -103,7 +103,7 @@ def test_errors(tmpdir, backend): # --- put_file --- with pytest.raises(audbackend.BackendError): - backend.put_file( + interface.put_file( local_file, remote_file, version, @@ -111,28 +111,28 @@ def test_errors(tmpdir, backend): # --- latest_version --- with pytest.raises(audbackend.BackendError): - backend.latest_version(remote_file) + interface.latest_version(remote_file) # --- ls --- with pytest.raises(audbackend.BackendError): - backend.ls('/') - assert backend.ls( + interface.ls('/') + assert interface.ls( '/', suppress_backend_errors=True, ) == [] # --- versions --- with pytest.raises(audbackend.BackendError): - backend.versions(remote_file) - assert backend.versions( + interface.versions(remote_file) + assert interface.versions( remote_file, suppress_backend_errors=True, ) == [] @pytest.mark.parametrize( - 'backend', - ['artifactory'], + 'interface', + [('artifactory', audbackend.interface.Versioned)], indirect=True, ) @pytest.mark.parametrize( @@ -203,15 +203,18 @@ def test_errors(tmpdir, backend): ), ] ) -def test_legacy_file_structure(tmpdir, backend, file, version, extensions, +def test_legacy_file_structure(tmpdir, interface, file, version, extensions, regex, expected): - backend._use_legacy_file_structure(extensions=extensions, regex=regex) + interface._use_legacy_file_structure(extensions=extensions, regex=regex) src_path = audeer.touch(audeer.path(tmpdir, 'tmp')) - backend.put_file(src_path, file, version) + interface.put_file(src_path, file, version) - url = f'{str(backend._repo.path)}{expected}' - assert backend._expand(backend._path_with_version(file, version)) == url - assert backend.ls(file) == [(file, version)] - assert backend.ls() == [(file, version)] + url = f'{str(interface.backend._repo.path)}{expected}' + url_expected = interface.backend._expand( + interface._path_with_version(file, version), + ) + assert url_expected == url + assert interface.ls(file) == [(file, version)] + assert interface.ls() == [(file, version)] diff --git a/tests/test_backend_base.py b/tests/test_backend_base.py new file mode 100644 index 00000000..1157ffbe --- /dev/null +++ b/tests/test_backend_base.py @@ -0,0 +1,83 @@ +import pytest + +import audbackend + + +@pytest.mark.parametrize( + 'paths, expected', + [ + (['/'], '/'), + (['/', ''], '/'), + (['/file'], '/file'), + (['/file/'], '/file/'), + (['/root', 'file'], '/root/file'), + (['/root', 'file/'], '/root/file/'), + (['/', 'root', None, '', 'file', ''], '/root/file'), + (['/', 'root', None, '', 'file', '/'], '/root/file/'), + (['/', 'root', None, '', 'file', '/', ''], '/root/file/'), + pytest.param( + [''], + None, + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + ['file'], + None, + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + ['sub/file'], + None, + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + ['', '/file'], + None, + marks=pytest.mark.xfail(raises=ValueError), + ), + ] +) +@pytest.mark.parametrize( + 'backend', + [ + audbackend.Backend('host', 'repository'), + ] +) +def test_join(paths, expected, backend): + assert backend.join(*paths) == expected + + +@pytest.mark.parametrize( + 'path, expected', + [ + ('/', ('/', '')), + ('/file', ('/', 'file')), + ('/root/', ('/root/', '')), + ('/root/file', ('/root/', 'file')), + ('/root/file/', ('/root/file/', '')), + ('//root///file', ('/root/', 'file')), + pytest.param( + '', + None, + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + 'file', + None, + marks=pytest.mark.xfail(raises=ValueError), + ), + pytest.param( + 'sub/file', + None, + marks=pytest.mark.xfail(raises=ValueError), + ), + ] +) +@pytest.mark.parametrize( + 'backend', + [ + audbackend.Backend('host', 'repository'), + ] +) +def test_split(path, expected, backend): + assert backend.split(path) == expected diff --git a/tests/test_filesystem.py b/tests/test_backend_filesystem.py similarity index 79% rename from tests/test_filesystem.py rename to tests/test_backend_filesystem.py index 228afb0c..d0097b79 100644 --- a/tests/test_filesystem.py +++ b/tests/test_backend_filesystem.py @@ -13,10 +13,9 @@ def _get_file( self, src_path: str, dst_path: str, - version: str, verbose: bool, ): - super()._get_file(src_path, dst_path, version, verbose) + super()._get_file(src_path, dst_path, verbose) # raise error after file was retrieved raise InterruptedError() @@ -29,11 +28,11 @@ def bad_file_system(): @pytest.mark.parametrize( - 'backend', - ['file-system'], + 'interface', + [('file-system', audbackend.interface.Versioned)], indirect=True, ) -def test_get_file_interrupt(tmpdir, bad_file_system, backend): +def test_get_file_interrupt(tmpdir, bad_file_system, interface): src_path = audeer.path(tmpdir, '~tmp') @@ -41,7 +40,7 @@ def test_get_file_interrupt(tmpdir, bad_file_system, backend): with open(src_path, 'w') as fp: fp.write('remote') checksum_remote = audeer.md5(src_path) - backend.put_file(src_path, '/file', '1.0.0') + interface.put_file(src_path, '/file', '1.0.0') # change content of local file with open(src_path, 'w') as fp: @@ -51,13 +50,13 @@ def test_get_file_interrupt(tmpdir, bad_file_system, backend): # try to read remote file, local file remains unchanged with pytest.raises(audbackend.BackendError): - backend.get_file('/file', src_path, '1.0.0') + interface.get_file('/file', src_path, '1.0.0') assert audeer.md5(src_path) == checksum_local @pytest.mark.parametrize( - 'backend', - ['file-system'], + 'interface', + [('file-system', audbackend.interface.Versioned)], indirect=True, ) @pytest.mark.parametrize( @@ -128,17 +127,20 @@ def test_get_file_interrupt(tmpdir, bad_file_system, backend): ), ] ) -def test_legacy_file_structure(tmpdir, backend, file, version, extensions, +def test_legacy_file_structure(tmpdir, interface, file, version, extensions, regex, expected): expected = expected.replace('/', os.path.sep) - backend._use_legacy_file_structure(extensions=extensions, regex=regex) + interface._use_legacy_file_structure(extensions=extensions, regex=regex) src_path = audeer.touch(audeer.path(tmpdir, 'tmp')) - backend.put_file(src_path, file, version) - - path = os.path.join(backend._root, expected) - assert backend._expand(backend._path_with_version(file, version)) == path - assert backend.ls(file) == [(file, version)] - assert backend.ls() == [(file, version)] + interface.put_file(src_path, file, version) + + path = os.path.join(interface.backend._root, expected) + path_expected = interface.backend._expand( + interface._path_with_version(file, version), + ) + assert path_expected == path + assert interface.ls(file) == [(file, version)] + assert interface.ls() == [(file, version)] diff --git a/tests/test_filesystem_only.py b/tests/test_backend_filesystem_only.py similarity index 67% rename from tests/test_filesystem_only.py rename to tests/test_backend_filesystem_only.py index 2fda0793..12617250 100644 --- a/tests/test_filesystem_only.py +++ b/tests/test_backend_filesystem_only.py @@ -5,4 +5,4 @@ # Check optional backends are not available with pytest.raises(AttributeError): - audbackend.Artifactory() + audbackend.Artifactory('https://host.com', 'repo') diff --git a/tests/test_interface_unversioned.py b/tests/test_interface_unversioned.py new file mode 100644 index 00000000..b185f9e2 --- /dev/null +++ b/tests/test_interface_unversioned.py @@ -0,0 +1,508 @@ +import datetime +import os +import platform +import re +import stat + +import pytest + +import audeer + +import audbackend + + +@pytest.fixture(scope='function', autouse=False) +def tree(tmpdir, request): + r"""Create file tree.""" + files = request.param + paths = [] + + for path in files: + if os.name == 'nt': + path = path.replace('/', os.path.sep) + if path.endswith(os.path.sep): + path = audeer.path(tmpdir, path) + path = audeer.mkdir(path) + path = path + os.path.sep + paths.append(path) + else: + path = audeer.path(tmpdir, path) + audeer.mkdir(os.path.dirname(path)) + path = audeer.touch(path) + paths.append(path) + + yield paths + + +@pytest.mark.parametrize( + 'tree, archive, files, tmp_root, expected', + [ + ( # empty + ['file.ext', 'dir/to/file.ext'], + '/archive.zip', + [], + None, + [], + ), + ( # single file + ['file.ext', 'dir/to/file.ext'], + '/archive.zip', + 'file.ext', + None, + ['file.ext'], + ), + ( # list + ['file.ext', 'dir/to/file.ext'], + '/archive.zip', + ['file.ext'], + None, + ['file.ext'], + ), + ( + ['file.ext', 'dir/to/file.ext'], + '/archive.zip', + ['file.ext', 'dir/to/file.ext'], + 'tmp', + ['file.ext', 'dir/to/file.ext'], + ), + ( # all files + ['file.ext', 'dir/to/file.ext'], + '/archive.zip', + None, + 'tmp', + ['dir/to/file.ext', 'file.ext'], + ), + ( # tar.gz + ['file.ext', 'dir/to/file.ext'], + '/archive.tar.gz', + None, + 'tmp', + ['dir/to/file.ext', 'file.ext'], + ), + ], + indirect=['tree'], +) +@pytest.mark.parametrize( + 'interface', + pytest.UNVERSIONED, + indirect=True, +) +def test_archive(tmpdir, tree, archive, files, tmp_root, interface, expected): + + if tmp_root is not None: + tmp_root = audeer.path(tmpdir, tmp_root) + + if os.name == 'nt': + expected = [file.replace('/', os.sep) for file in expected] + + # if a tmp_root is given but does not exist, + # put_archive() should fail + if tmp_root is not None: + if os.path.exists(tmp_root): + os.removedirs(tmp_root) + with pytest.raises(FileNotFoundError): + interface.put_archive( + tmpdir, + archive, + files=files, + tmp_root=tmp_root, + ) + audeer.mkdir(tmp_root) + + interface.put_archive( + tmpdir, + archive, + files=files, + tmp_root=tmp_root, + ) + # operation will be skipped + interface.put_archive( + tmpdir, + archive, + files=files, + tmp_root=tmp_root, + ) + assert interface.exists(archive) + + # if a tmp_root is given but does not exist, + # get_archive() should fail + if tmp_root is not None: + if os.path.exists(tmp_root): + os.removedirs(tmp_root) + with pytest.raises(FileNotFoundError): + interface.get_archive( + archive, + tmpdir, + tmp_root=tmp_root, + ) + audeer.mkdir(tmp_root) + + assert interface.get_archive( + archive, + tmpdir, + tmp_root=tmp_root, + ) == expected + + +@pytest.mark.parametrize( + 'interface', + pytest.UNVERSIONED, + indirect=True, +) +def test_errors(tmpdir, interface): + + # Ensure we have one file and one archive published on the backend + archive = '/archive.zip' + local_file = 'file.txt' + local_path = audeer.touch(audeer.path(tmpdir, local_file)) + local_folder = audeer.mkdir(audeer.path(tmpdir, 'folder')) + remote_file = f'/{local_file}' + interface.put_file(local_path, remote_file) + interface.put_archive(tmpdir, archive, files=[local_file]) + + # Create local read-only file and folder + file_read_only = audeer.touch(audeer.path(tmpdir, 'read-only-file.txt')) + os.chmod(file_read_only, stat.S_IRUSR) + folder_read_only = audeer.mkdir(audeer.path(tmpdir, 'read-only-folder')) + os.chmod(folder_read_only, stat.S_IRUSR) + + # Invalid file names / versions and error messages + file_invalid_path = 'invalid/path.txt' + error_invalid_path = re.escape( + f"Invalid backend path '{file_invalid_path}', " + f"must start with '/'." + ) + file_invalid_char = '/invalid/char.txt?' + error_invalid_char = re.escape( + f"Invalid backend path '{file_invalid_char}', " + f"does not match '[A-Za-z0-9/._-]+'." + ) + error_backend = ( + 'An exception was raised by the backend, ' + 'please see stack trace for further information.' + ) + error_read_only_folder = ( + f"Permission denied: '{os.path.join(folder_read_only, local_file)}'" + ) + error_read_only_file = ( + f"Permission denied: '{file_read_only}'" + ) + if platform.system() == 'Windows': + error_is_a_folder = "Is a directory: " + else: + error_is_a_folder = f"Is a directory: '{local_folder}'" + if platform.system() == 'Windows': + error_not_a_folder = "Not a directory: " + else: + error_not_a_folder = f"Not a directory: '{local_path}'" + + # --- checksum --- + # `path` missing + with pytest.raises(audbackend.BackendError, match=error_backend): + interface.checksum('/missing.txt') + # `path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.checksum(file_invalid_char) + + # --- exists --- + # `path` without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.exists(file_invalid_path) + # `path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.exists(file_invalid_char) + + # --- get_archive --- + # `src_path` missing + with pytest.raises(audbackend.BackendError, match=error_backend): + interface.get_archive('/missing.txt', tmpdir) + # `src_path` without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.get_archive(file_invalid_path, tmpdir) + # `src_path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.get_archive(file_invalid_char, tmpdir) + # `tmp_root` does not exist + if platform.system() == 'Windows': + error_msg = ( + "The system cannot find the path specified: 'non-existing..." + ) + else: + error_msg = "No such file or directory: 'non-existing/..." + with pytest.raises(FileNotFoundError, match=error_msg): + interface.get_archive(archive, tmpdir, tmp_root='non-existing') + # extension of `src_path` is not supported + error_msg = 'You can only extract ZIP and TAR.GZ files, ...' + interface.put_file( + audeer.touch(audeer.path(tmpdir, 'archive.bad')), + '/archive.bad', + ) + with pytest.raises(RuntimeError, match=error_msg): + interface.get_archive('/archive.bad', tmpdir) + # `src_path` is a malformed archive + error_msg = 'Broken archive: ' + interface.put_file( + audeer.touch(audeer.path(tmpdir, 'malformed.zip')), + '/malformed.zip', + ) + with pytest.raises(RuntimeError, match=error_msg): + interface.get_archive('/malformed.zip', tmpdir) + # no write permissions to `dst_root` + if not platform.system() == 'Windows': + # Currently we don't know how to provoke permission error on Windows + with pytest.raises(PermissionError, match=error_read_only_folder): + interface.get_archive(archive, folder_read_only) + # `dst_root` is not a directory + with pytest.raises(NotADirectoryError, match=error_not_a_folder): + interface.get_archive(archive, local_path) + + # --- get_file --- + # `src_path` missing + with pytest.raises(audbackend.BackendError, match=error_backend): + interface.get_file('/missing.txt', 'missing.txt') + # `src_path` without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.get_file(file_invalid_path, tmpdir) + # `src_path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.get_file(file_invalid_char, tmpdir) + # no write permissions to `dst_path` + if not platform.system() == 'Windows': + # Currently we don't know how to provoke permission error on Windows + with pytest.raises(PermissionError, match=error_read_only_file): + interface.get_file(remote_file, file_read_only) + dst_path = audeer.path(folder_read_only, 'file.txt') + with pytest.raises(PermissionError, match=error_read_only_folder): + interface.get_file(remote_file, dst_path) + # `dst_path` is an existing folder + with pytest.raises(IsADirectoryError, match=error_is_a_folder): + interface.get_file(remote_file, local_folder) + + # --- join --- + # joined path without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.join(file_invalid_path, local_file) + # joined path contains invalid char + with pytest.raises(ValueError, match=error_invalid_char): + interface.join(file_invalid_char, local_file) + + # --- ls --- + # `path` does not exist + with pytest.raises(audbackend.BackendError, match=error_backend): + interface.ls('/missing/') + interface.ls('/missing/', suppress_backend_errors=True) + with pytest.raises(audbackend.BackendError, match=error_backend): + interface.ls('/missing.txt') + interface.ls('/missing.txt', suppress_backend_errors=True) + remote_file_with_wrong_ext = audeer.replace_file_extension( + remote_file, + 'missing', + ) + with pytest.raises(audbackend.BackendError, match=error_backend): + interface.ls(remote_file_with_wrong_ext) + interface.ls(remote_file_with_wrong_ext, suppress_backend_errors=True) + # joined path without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.ls(file_invalid_path) + # `path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.ls(file_invalid_char) + + # --- put_archive --- + # `src_root` missing + error_msg = 'No such file or directory: ...' + with pytest.raises(FileNotFoundError, match=error_msg): + interface.put_archive( + audeer.path(tmpdir, '/missing/'), + archive, + files=local_file, + ) + # `src_root` is not a directory + with pytest.raises(NotADirectoryError, match=error_not_a_folder): + interface.put_archive(local_path, archive) + # `files` missing + error_msg = 'No such file or directory: ...' + with pytest.raises(FileNotFoundError, match=error_msg): + interface.put_archive(tmpdir, archive, files='missing.txt') + # `dst_path` without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.put_archive( + tmpdir, + file_invalid_path, + files=local_file, + ) + # `dst_path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.put_archive( + tmpdir, + file_invalid_char, + files=local_file, + ) + # extension of `dst_path` is not supported + error_msg = 'You can only create a ZIP or TAR.GZ archive, not ...' + with pytest.raises(RuntimeError, match=error_msg): + interface.put_archive(tmpdir, '/archive.bad', files=local_file) + + # --- put_file --- + # `src_path` does not exists + error_msg = 'No such file or directory: ...' + with pytest.raises(FileNotFoundError, match=error_msg): + interface.put_file( + audeer.path(tmpdir, 'missing.txt'), + remote_file, + ) + # `src_path` is a folder + with pytest.raises(IsADirectoryError, match=error_is_a_folder): + interface.put_file(local_folder, remote_file) + # `dst_path` without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.put_file(local_path, file_invalid_path) + # `dst_path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.put_file(local_path, file_invalid_char) + + # --- remove_file --- + # `path` does not exists + with pytest.raises(audbackend.BackendError, match=error_backend): + interface.remove_file('/missing.txt') + # `path` without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.remove_file(file_invalid_path) + # `path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.remove_file(file_invalid_char) + + # --- split --- + # `path` without leading '/' + with pytest.raises(ValueError, match=error_invalid_path): + interface.split(file_invalid_path) + # `path` contains invalid character + with pytest.raises(ValueError, match=error_invalid_char): + interface.split(file_invalid_char) + + +@pytest.mark.parametrize( + 'path', + [ + '/file.txt', + '/folder/test.txt', + ] +) +@pytest.mark.parametrize( + 'interface', + pytest.UNVERSIONED, + indirect=True, +) +def test_exists(tmpdir, path, interface): + + src_path = audeer.path(tmpdir, '~') + audeer.touch(src_path) + + assert not interface.exists(path) + interface.put_file(src_path, path) + assert interface.exists(path) + + +@pytest.mark.parametrize( + 'src_path, dst_path', + [ + ( + 'file', + '/file', + ), + ( + 'file.ext', + '/file.ext', + ), + ( + os.path.join('dir', 'to', 'file.ext'), + '/dir/to/file.ext', + ), + ( + os.path.join('dir.to', 'file.ext'), + '/dir.to/file.ext', + ), + ], +) +@pytest.mark.parametrize( + 'interface, owner', + [(x, x[0]) for x in pytest.UNVERSIONED], + indirect=True, +) +def test_file(tmpdir, src_path, dst_path, interface, owner): + + src_path = audeer.path(tmpdir, src_path) + audeer.mkdir(os.path.dirname(src_path)) + audeer.touch(src_path) + + assert not interface.exists(dst_path) + interface.put_file(src_path, dst_path) + # operation will be skipped + interface.put_file(src_path, dst_path) + assert interface.exists(dst_path) + + interface.get_file(dst_path, src_path) + assert os.path.exists(src_path) + assert interface.checksum(dst_path) == audeer.md5(src_path) + assert interface.owner(dst_path) == owner + date = datetime.datetime.today().strftime('%Y-%m-%d') + assert interface.date(dst_path) == date + + interface.remove_file(dst_path) + assert not interface.exists(dst_path) + + +@pytest.mark.parametrize( + 'interface', + pytest.UNVERSIONED, + indirect=True, +) +def test_ls(tmpdir, interface): + + assert interface.ls() == [] + assert interface.ls('/') == [] + + root = [ + '/file.bar', + '/file.foo', + ] + root_foo = [ + '/file.foo', + ] + root_bar = [ + '/file.bar', + ] + sub = [ + '/sub/file.foo', + ] + hidden = [ + '/.sub/.file.foo', + ] + + # create content + + tmp_file = os.path.join(tmpdir, '~') + for path in root + sub + hidden: + audeer.touch(tmp_file) + interface.put_file( + tmp_file, + path, + ) + + # test + + for path, pattern, expected in [ + ('/', None, root + sub + hidden), + ('/', '*.foo', root_foo + sub + hidden), + ('/sub/', None, sub), + ('/sub/', '*.bar', []), + ('/sub/', 'file.*', sub), + ('/.sub/', None, hidden), + ('/file.bar', None, root_bar), + ('/sub/file.foo', None, sub), + ('/.sub/.file.foo', None, hidden), + ]: + assert interface.ls( + path, + pattern=pattern, + ) == sorted(expected) diff --git a/tests/test_backend.py b/tests/test_interface_versioned.py similarity index 68% rename from tests/test_backend.py rename to tests/test_interface_versioned.py index 17230a51..8f0b51d6 100644 --- a/tests/test_backend.py +++ b/tests/test_interface_versioned.py @@ -83,11 +83,11 @@ def tree(tmpdir, request): indirect=['tree'], ) @pytest.mark.parametrize( - 'backend', - pytest.BACKENDS, + 'interface', + pytest.VERSIONED, indirect=True, ) -def test_archive(tmpdir, tree, archive, files, tmp_root, backend, expected): +def test_archive(tmpdir, tree, archive, files, tmp_root, interface, expected): version = '1.0.0' @@ -103,7 +103,7 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, backend, expected): if os.path.exists(tmp_root): os.removedirs(tmp_root) with pytest.raises(FileNotFoundError): - backend.put_archive( + interface.put_archive( tmpdir, archive, version, @@ -112,7 +112,7 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, backend, expected): ) audeer.mkdir(tmp_root) - backend.put_archive( + interface.put_archive( tmpdir, archive, version, @@ -120,14 +120,14 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, backend, expected): tmp_root=tmp_root, ) # operation will be skipped - backend.put_archive( + interface.put_archive( tmpdir, archive, version, files=files, tmp_root=tmp_root, ) - assert backend.exists(archive, version) + assert interface.exists(archive, version) # if a tmp_root is given but does not exist, # get_archive() should fail @@ -135,7 +135,7 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, backend, expected): if os.path.exists(tmp_root): os.removedirs(tmp_root) with pytest.raises(FileNotFoundError): - backend.get_archive( + interface.get_archive( archive, tmpdir, version, @@ -143,7 +143,7 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, backend, expected): ) audeer.mkdir(tmp_root) - assert backend.get_archive( + assert interface.get_archive( archive, tmpdir, version, @@ -152,11 +152,11 @@ def test_archive(tmpdir, tree, archive, files, tmp_root, backend, expected): @pytest.mark.parametrize( - 'backend', - pytest.BACKENDS, + 'interface', + pytest.VERSIONED, indirect=True, ) -def test_errors(tmpdir, backend): +def test_errors(tmpdir, interface): # Ensure we have one file and one archive published on the backend archive = '/archive.zip' @@ -165,8 +165,8 @@ def test_errors(tmpdir, backend): local_folder = audeer.mkdir(audeer.path(tmpdir, 'folder')) remote_file = f'/{local_file}' version = '1.0.0' - backend.put_file(local_path, remote_file, version) - backend.put_archive(tmpdir, archive, version, files=[local_file]) + interface.put_file(local_path, remote_file, version) + interface.put_archive(tmpdir, archive, version, files=[local_file]) # Create local read-only file and folder file_read_only = audeer.touch(audeer.path(tmpdir, 'read-only-file.txt')) @@ -214,44 +214,44 @@ def test_errors(tmpdir, backend): # --- checksum --- # `path` missing with pytest.raises(audbackend.BackendError, match=error_backend): - backend.checksum('/missing.txt', version) + interface.checksum('/missing.txt', version) # `path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.checksum(file_invalid_char, version) + interface.checksum(file_invalid_char, version) # invalid version with pytest.raises(ValueError, match=error_empty_version): - backend.checksum(remote_file, empty_version) + interface.checksum(remote_file, empty_version) with pytest.raises(ValueError, match=error_invalid_version): - backend.checksum(remote_file, invalid_version) + interface.checksum(remote_file, invalid_version) # --- exists --- # `path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.exists(file_invalid_path, version) + interface.exists(file_invalid_path, version) # `path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.exists(file_invalid_char, version) + interface.exists(file_invalid_char, version) # invalid version with pytest.raises(ValueError, match=error_empty_version): - backend.exists(remote_file, empty_version) + interface.exists(remote_file, empty_version) with pytest.raises(ValueError, match=error_invalid_version): - backend.exists(remote_file, invalid_version) + interface.exists(remote_file, invalid_version) # --- get_archive --- # `src_path` missing with pytest.raises(audbackend.BackendError, match=error_backend): - backend.get_archive('/missing.txt', tmpdir, version) + interface.get_archive('/missing.txt', tmpdir, version) # `src_path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.get_archive(file_invalid_path, tmpdir, version) + interface.get_archive(file_invalid_path, tmpdir, version) # `src_path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.get_archive(file_invalid_char, tmpdir, version) + interface.get_archive(file_invalid_char, tmpdir, version) # invalid version with pytest.raises(ValueError, match=error_empty_version): - backend.get_archive(archive, tmpdir, empty_version) + interface.get_archive(archive, tmpdir, empty_version) with pytest.raises(ValueError, match=error_invalid_version): - backend.get_archive(archive, tmpdir, invalid_version) + interface.get_archive(archive, tmpdir, invalid_version) # `tmp_root` does not exist if platform.system() == 'Windows': error_msg = ( @@ -260,104 +260,112 @@ def test_errors(tmpdir, backend): else: error_msg = "No such file or directory: 'non-existing/..." with pytest.raises(FileNotFoundError, match=error_msg): - backend.get_archive(archive, tmpdir, version, tmp_root='non-existing') + interface.get_archive( + archive, + tmpdir, + version, + tmp_root='non-existing', + ) # extension of `src_path` is not supported error_msg = 'You can only extract ZIP and TAR.GZ files, ...' - backend.put_file( + interface.put_file( audeer.touch(audeer.path(tmpdir, 'archive.bad')), '/archive.bad', version, ) with pytest.raises(RuntimeError, match=error_msg): - backend.get_archive('/archive.bad', tmpdir, version) + interface.get_archive('/archive.bad', tmpdir, version) # `src_path` is a malformed archive error_msg = 'Broken archive: ' - backend.put_file( + interface.put_file( audeer.touch(audeer.path(tmpdir, 'malformed.zip')), '/malformed.zip', version, ) with pytest.raises(RuntimeError, match=error_msg): - backend.get_archive('/malformed.zip', tmpdir, version) + interface.get_archive('/malformed.zip', tmpdir, version) # no write permissions to `dst_root` if not platform.system() == 'Windows': # Currently we don't know how to provoke permission error on Windows with pytest.raises(PermissionError, match=error_read_only_folder): - backend.get_archive(archive, folder_read_only, version) + interface.get_archive(archive, folder_read_only, version) # `dst_root` is not a directory with pytest.raises(NotADirectoryError, match=error_not_a_folder): - backend.get_archive(archive, local_path, version) + interface.get_archive(archive, local_path, version) # --- get_file --- # `src_path` missing with pytest.raises(audbackend.BackendError, match=error_backend): - backend.get_file('/missing.txt', 'missing.txt', version) + interface.get_file('/missing.txt', 'missing.txt', version) # `src_path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.get_file(file_invalid_path, tmpdir, version) + interface.get_file(file_invalid_path, tmpdir, version) # `src_path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.get_file(file_invalid_char, tmpdir, version) + interface.get_file(file_invalid_char, tmpdir, version) # invalid version with pytest.raises(ValueError, match=error_empty_version): - backend.get_file(remote_file, local_file, empty_version) + interface.get_file(remote_file, local_file, empty_version) with pytest.raises(ValueError, match=error_invalid_version): - backend.get_file(remote_file, local_file, invalid_version) + interface.get_file(remote_file, local_file, invalid_version) # no write permissions to `dst_path` if not platform.system() == 'Windows': # Currently we don't know how to provoke permission error on Windows with pytest.raises(PermissionError, match=error_read_only_file): - backend.get_file(remote_file, file_read_only, version) + interface.get_file(remote_file, file_read_only, version) dst_path = audeer.path(folder_read_only, 'file.txt') with pytest.raises(PermissionError, match=error_read_only_folder): - backend.get_file(remote_file, dst_path, version) + interface.get_file(remote_file, dst_path, version) # `dst_path` is an existing folder with pytest.raises(IsADirectoryError, match=error_is_a_folder): - backend.get_file(remote_file, local_folder, version) + interface.get_file(remote_file, local_folder, version) # --- join --- # joined path without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.join(file_invalid_path, local_file) + interface.join(file_invalid_path, local_file) # joined path contains invalid char with pytest.raises(ValueError, match=error_invalid_char): - backend.join(file_invalid_char, local_file) + interface.join(file_invalid_char, local_file) # --- latest_version --- # `path` missing with pytest.raises(audbackend.BackendError, match=error_backend): - backend.latest_version('/missing.txt') + interface.latest_version('/missing.txt') # joined path without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.latest_version(file_invalid_path) + interface.latest_version(file_invalid_path) # `path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.latest_version(file_invalid_char) + interface.latest_version(file_invalid_char) # --- ls --- # `path` does not exist with pytest.raises(audbackend.BackendError, match=error_backend): - backend.ls('/missing/') + interface.ls('/missing/') + interface.ls('/missing/', suppress_backend_errors=True) with pytest.raises(audbackend.BackendError, match=error_backend): - backend.ls('/missing.txt') + interface.ls('/missing.txt') + interface.ls('/missing.txt', suppress_backend_errors=True) remote_file_with_wrong_ext = audeer.replace_file_extension( remote_file, 'missing', ) with pytest.raises(audbackend.BackendError, match=error_backend): - backend.ls(remote_file_with_wrong_ext) + interface.ls(remote_file_with_wrong_ext) + interface.ls(remote_file_with_wrong_ext, suppress_backend_errors=True) # joined path without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.ls(file_invalid_path) + interface.ls(file_invalid_path) # `path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.ls(file_invalid_char) + interface.ls(file_invalid_char) # --- put_archive --- # `src_root` missing error_msg = 'No such file or directory: ...' with pytest.raises(FileNotFoundError, match=error_msg): - backend.put_archive( + interface.put_archive( audeer.path(tmpdir, '/missing/'), archive, version, @@ -365,14 +373,14 @@ def test_errors(tmpdir, backend): ) # `src_root` is not a directory with pytest.raises(NotADirectoryError, match=error_not_a_folder): - backend.put_archive(local_path, archive, version) + interface.put_archive(local_path, archive, version) # `files` missing error_msg = 'No such file or directory: ...' with pytest.raises(FileNotFoundError, match=error_msg): - backend.put_archive(tmpdir, archive, version, files='missing.txt') + interface.put_archive(tmpdir, archive, version, files='missing.txt') # `dst_path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.put_archive( + interface.put_archive( tmpdir, file_invalid_path, version, @@ -380,7 +388,7 @@ def test_errors(tmpdir, backend): ) # `dst_path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.put_archive( + interface.put_archive( tmpdir, file_invalid_char, version, @@ -388,69 +396,74 @@ def test_errors(tmpdir, backend): ) # invalid version with pytest.raises(ValueError, match=error_empty_version): - backend.put_archive(tmpdir, archive, empty_version) + interface.put_archive(tmpdir, archive, empty_version) with pytest.raises(ValueError, match=error_invalid_version): - backend.put_archive(tmpdir, archive, invalid_version) + interface.put_archive(tmpdir, archive, invalid_version) # extension of `dst_path` is not supported error_msg = 'You can only create a ZIP or TAR.GZ archive, not ...' with pytest.raises(RuntimeError, match=error_msg): - backend.put_archive(tmpdir, '/archive.bad', version, files=local_file) + interface.put_archive( + tmpdir, + '/archive.bad', + version, + files=local_file, + ) # --- put_file --- # `src_path` does not exists error_msg = 'No such file or directory: ...' with pytest.raises(FileNotFoundError, match=error_msg): - backend.put_file( + interface.put_file( audeer.path(tmpdir, 'missing.txt'), remote_file, version, ) # `src_path` is a folder with pytest.raises(IsADirectoryError, match=error_is_a_folder): - backend.put_file(local_folder, remote_file, version) + interface.put_file(local_folder, remote_file, version) # `dst_path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.put_file(local_path, file_invalid_path, version) + interface.put_file(local_path, file_invalid_path, version) # `dst_path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.put_file(local_path, file_invalid_char, version) + interface.put_file(local_path, file_invalid_char, version) # invalid version with pytest.raises(ValueError, match=error_empty_version): - backend.put_file(local_path, remote_file, empty_version) + interface.put_file(local_path, remote_file, empty_version) with pytest.raises(ValueError, match=error_invalid_version): - backend.put_file(local_path, remote_file, invalid_version) + interface.put_file(local_path, remote_file, invalid_version) # --- remove_file --- # `path` does not exists with pytest.raises(audbackend.BackendError, match=error_backend): - backend.remove_file('/missing.txt', version) + interface.remove_file('/missing.txt', version) # `path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.remove_file(file_invalid_path, version) + interface.remove_file(file_invalid_path, version) # `path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.remove_file(file_invalid_char, version) + interface.remove_file(file_invalid_char, version) # invalid version with pytest.raises(ValueError, match=error_empty_version): - backend.remove_file(remote_file, empty_version) + interface.remove_file(remote_file, empty_version) with pytest.raises(ValueError, match=error_invalid_version): - backend.remove_file(remote_file, invalid_version) + interface.remove_file(remote_file, invalid_version) # --- split --- # `path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.split(file_invalid_path) + interface.split(file_invalid_path) # `path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.split(file_invalid_char) + interface.split(file_invalid_char) # --- versions --- # `path` without leading '/' with pytest.raises(ValueError, match=error_invalid_path): - backend.versions(file_invalid_path) + interface.versions(file_invalid_path) # `path` contains invalid character with pytest.raises(ValueError, match=error_invalid_char): - backend.versions(file_invalid_char) + interface.versions(file_invalid_char) @pytest.mark.parametrize( @@ -461,18 +474,18 @@ def test_errors(tmpdir, backend): ] ) @pytest.mark.parametrize( - 'backend', - pytest.BACKENDS, + 'interface', + pytest.VERSIONED, indirect=True, ) -def test_exists(tmpdir, path, version, backend): +def test_exists(tmpdir, path, version, interface): src_path = audeer.path(tmpdir, '~') audeer.touch(src_path) - assert not backend.exists(path, version) - backend.put_file(src_path, path, version) - assert backend.exists(path, version) + assert not interface.exists(path, version) + interface.put_file(src_path, path, version) + assert interface.exists(path, version) @pytest.mark.parametrize( @@ -501,42 +514,42 @@ def test_exists(tmpdir, path, version, backend): ], ) @pytest.mark.parametrize( - 'backend, owner', - [(name, name) for name in pytest.BACKENDS], + 'interface, owner', + [(x, x[0]) for x in pytest.VERSIONED], indirect=True, ) -def test_file(tmpdir, src_path, dst_path, version, backend, owner): +def test_file(tmpdir, src_path, dst_path, version, interface, owner): src_path = audeer.path(tmpdir, src_path) audeer.mkdir(os.path.dirname(src_path)) audeer.touch(src_path) - assert not backend.exists(dst_path, version) - backend.put_file(src_path, dst_path, version) + assert not interface.exists(dst_path, version) + interface.put_file(src_path, dst_path, version) # operation will be skipped - backend.put_file(src_path, dst_path, version) - assert backend.exists(dst_path, version) + interface.put_file(src_path, dst_path, version) + assert interface.exists(dst_path, version) - backend.get_file(dst_path, src_path, version) + interface.get_file(dst_path, src_path, version) assert os.path.exists(src_path) - assert backend.checksum(dst_path, version) == audeer.md5(src_path) - assert backend.owner(dst_path, version) == owner + assert interface.checksum(dst_path, version) == audeer.md5(src_path) + assert interface.owner(dst_path, version) == owner date = datetime.datetime.today().strftime('%Y-%m-%d') - assert backend.date(dst_path, version) == date + assert interface.date(dst_path, version) == date - backend.remove_file(dst_path, version) - assert not backend.exists(dst_path, version) + interface.remove_file(dst_path, version) + assert not interface.exists(dst_path, version) @pytest.mark.parametrize( - 'backend', - pytest.BACKENDS, + 'interface', + pytest.VERSIONED, indirect=True, ) -def test_ls(tmpdir, backend): +def test_ls(tmpdir, interface): - assert backend.ls() == [] - assert backend.ls('/') == [] + assert interface.ls() == [] + assert interface.ls('/') == [] root = [ ('/file.bar', '1.0.0'), @@ -577,7 +590,7 @@ def test_ls(tmpdir, backend): tmp_file = os.path.join(tmpdir, '~') for path, version in root + sub + hidden: audeer.touch(tmp_file) - backend.put_file( + interface.put_file( tmp_file, path, version, @@ -605,93 +618,13 @@ def test_ls(tmpdir, backend): ('/.sub/.file.foo', False, None, hidden), ('/.sub/.file.foo', True, None, hidden_latest), ]: - assert backend.ls( + assert interface.ls( path, latest_version=latest, pattern=pattern, ) == sorted(expected) -@pytest.mark.parametrize( - 'paths, expected', - [ - (['/'], '/'), - (['/', ''], '/'), - (['/file'], '/file'), - (['/file/'], '/file/'), - (['/root', 'file'], '/root/file'), - (['/root', 'file/'], '/root/file/'), - (['/', 'root', None, '', 'file', ''], '/root/file'), - (['/', 'root', None, '', 'file', '/'], '/root/file/'), - (['/', 'root', None, '', 'file', '/', ''], '/root/file/'), - pytest.param( - [''], - None, - marks=pytest.mark.xfail(raises=ValueError), - ), - pytest.param( - ['file'], - None, - marks=pytest.mark.xfail(raises=ValueError), - ), - pytest.param( - ['sub/file'], - None, - marks=pytest.mark.xfail(raises=ValueError), - ), - pytest.param( - ['', '/file'], - None, - marks=pytest.mark.xfail(raises=ValueError), - ), - ] -) -@pytest.mark.parametrize( - 'backend', - [ - audbackend.Backend('host', 'repository'), - ] -) -def test_join(paths, expected, backend): - assert backend.join(*paths) == expected - - -@pytest.mark.parametrize( - 'path, expected', - [ - ('/', ('/', '')), - ('/file', ('/', 'file')), - ('/root/', ('/root/', '')), - ('/root/file', ('/root/', 'file')), - ('/root/file/', ('/root/file/', '')), - ('//root///file', ('/root/', 'file')), - pytest.param( - '', - None, - marks=pytest.mark.xfail(raises=ValueError), - ), - pytest.param( - 'file', - None, - marks=pytest.mark.xfail(raises=ValueError), - ), - pytest.param( - 'sub/file', - None, - marks=pytest.mark.xfail(raises=ValueError), - ), - ] -) -@pytest.mark.parametrize( - 'backend', - [ - audbackend.Backend('host', 'repository'), - ] -) -def test_split(path, expected, backend): - assert backend.split(path) == expected - - @pytest.mark.parametrize( 'dst_path', [ @@ -700,35 +633,35 @@ def test_split(path, expected, backend): ] ) @pytest.mark.parametrize( - 'backend', - pytest.BACKENDS, + 'interface', + pytest.VERSIONED, indirect=True, ) -def test_versions(tmpdir, dst_path, backend): +def test_versions(tmpdir, dst_path, interface): src_path = audeer.path(tmpdir, '~') audeer.touch(src_path) # empty backend with pytest.raises(audbackend.BackendError): - backend.versions(dst_path) - assert not backend.versions(dst_path, suppress_backend_errors=True) + interface.versions(dst_path) + assert not interface.versions(dst_path, suppress_backend_errors=True) with pytest.raises(audbackend.BackendError): - backend.latest_version(dst_path) + interface.latest_version(dst_path) # v1 - backend.put_file(src_path, dst_path, '1.0.0') - assert backend.versions(dst_path) == ['1.0.0'] - assert backend.latest_version(dst_path) == '1.0.0' + interface.put_file(src_path, dst_path, '1.0.0') + assert interface.versions(dst_path) == ['1.0.0'] + assert interface.latest_version(dst_path) == '1.0.0' # v2 - backend.put_file(src_path, dst_path, '2.0.0') - assert backend.versions(dst_path) == ['1.0.0', '2.0.0'] - assert backend.latest_version(dst_path) == '2.0.0' + interface.put_file(src_path, dst_path, '2.0.0') + assert interface.versions(dst_path) == ['1.0.0', '2.0.0'] + assert interface.latest_version(dst_path) == '2.0.0' # v3 with a different extension other_ext = 'other' other_remote_file = audeer.replace_file_extension(dst_path, other_ext) - backend.put_file(src_path, other_remote_file, '3.0.0') - assert backend.versions(dst_path) == ['1.0.0', '2.0.0'] - assert backend.latest_version(dst_path) == '2.0.0' + interface.put_file(src_path, other_remote_file, '3.0.0') + assert interface.versions(dst_path) == ['1.0.0', '2.0.0'] + assert interface.latest_version(dst_path) == '2.0.0'