EOS-27247: Move shared_storage from utils repo to experiments / Remov…

…e shared storage framework (#92)

* Moved shared_storage from utils repo to experiments

Signed-off-by: Parayya Vastrad <parayya.vastrad@seagate.com>

* Added directory structure

Signed-off-by: Parayya Vastrad <parayya.vastrad@seagate.com>

* Updated docs

Signed-off-by: Parayya Vastrad <parayya.vastrad@seagate.com>

Utils/shared_storage/docs/docs.md

@@ -0,0 +1,27 @@
+Shared storage provides a global namespace across the CORTX cluster. The global name space would come from clustered file system like Glusterfs
+
+Get Shared Storage path
+
+This will provide the shared storage path (global namespace) available on the node/cluster.
+
+API Specification -
+
+from cortx.utils.shared_storage import Storage
+
+# name - the directory name required to be created inside shared path (mount-point), if none, returns the shared path/mountpoint
+# exist_ok - If set to False, SharedStorageError is raised if directory name passed in already present.
+
+# path will be none if no shared storage available
+
+path = Storage.get_path(name=None, exist_ok=True)
+
+Example -
+
+from cortx.utils.shared_storage import Storage
+
+# return the mount point/shared path as is:
+path = Storage.get_path()
+
+# creates a directory named 'dir_name' in shared path and return its path.
+path = Storage.get_path(name='dir_name', exist_ok=True)
+

Utils/shared_storage/objective/objective.md

@@ -0,0 +1 @@
+Shared storage is more of an adapter type class and it can support more than one type of shared storage.

Utils/shared_storage/src/__init__.py

@@ -0,0 +1,30 @@
+# CORTX Python common library.
+# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+
+__title__ = 'shared_storage'
+
+from cortx.utils.shared_storage.error import SharedStorageError
+from cortx.utils.shared_storage.shared_storage_agent import SharedStorageAgent
+from cortx.utils.shared_storage.shared_storage import Storage
+
+__doc__ = """
+Shared storage framework
+
+This framework is a tool to fetch the shared storage available in the environment.
+It fetches the shared path from a conf file and returns it to the caller. 
+
+module: Storage"""
+
+__all__ = [SharedStorageError, SharedStorageAgent, Storage]

Utils/shared_storage/src/error.py

@@ -0,0 +1,23 @@
+# CORTX Python common library.
+# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+
+from cortx.utils.errors import UtilsError
+
+
+class SharedStorageError(UtilsError):
+    """ Generic Exception with error code and output. """
+
+    def __init__(self, rc, message, *args):
+        super().__init__(rc, message, *args)

Utils/shared_storage/src/shared_storage.py

@@ -0,0 +1,55 @@
+# CORTX Python common library.
+# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+
+import errno
+import os
+
+from cortx.utils.conf_store import Conf
+from cortx.utils.shared_storage import SharedStorageError
+from cortx.utils.shared_storage.shared_storage_agent import SharedStorageFactory
+from cortx.utils.common import CortxConf
+
+class Storage:
+
+    """ Shared Storage Framework over various types of Shared Storages  """
+
+    def __init__(self, cluster_conf):
+        """ Initialize and load shared storage backend """
+        CortxConf.init(cluster_conf=cluster_conf)
+        self.shared_storage_url = CortxConf.get('support>shared_path')
+        if self.shared_storage_url is not None:
+            self.shared_storage_agent = SharedStorageFactory.get_instance( \
+                self.shared_storage_url)
+
+    @staticmethod
+    def get_path(name: str = None, exist_ok: bool = True,
+        cluster_conf: str = 'yaml:///etc/cortx/cluster.conf') -> str:
+        """ return shared storage mountpoint """
+
+        storage = Storage(cluster_conf=cluster_conf)
+        if storage.shared_storage_url is None:
+            return None
+
+        shared_path = storage.shared_storage_agent.get_path()
+        if name:
+            try:
+                spec_path = os.path.join(shared_path, name)
+                os.makedirs(spec_path, exist_ok = exist_ok)
+                shared_path = spec_path
+            except OSError as e:
+                raise SharedStorageError(errno.EINVAL, \
+                    "dir already exists, (use exist_ok as True) %s" % e)
+
+        return shared_path

Utils/shared_storage/src/shared_storage_agent.py

@@ -0,0 +1,70 @@
+# CORTX Python common library.
+# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+
+import inspect
+import errno
+from urllib.parse import urlparse
+
+from cortx.utils import errors
+from cortx.utils.shared_storage import SharedStorageError
+
+class SharedStorageAgent:
+
+    """ A common interface over all shared storage agents """
+
+    def _fetch_path(self):
+        """ fetch path from source """
+        raise SharedStorageError(errno.EINVAL, \
+                "_fetch_path not implemented")
+
+    def get_path(self):
+        """ return fetched path of shared storage mountpoint """
+        shared_path = self._fetch_path()
+        return shared_path
+
+
+class SharedStorageFactory:
+
+    """ Factory class for shared storage types """
+
+    _storages = {}
+
+    @staticmethod
+    def get_instance(shared_storage_url: str) -> SharedStorageAgent:
+        """ Obtain instance of SharedStorageAgent for given file_type """
+
+        try:
+            url_spec = urlparse(shared_storage_url)
+        except Exception as e:
+            raise SharedStorageError(errno.EINVAL, \
+                "Invalid path: %s. %s", shared_storage_url, e)
+        storage_type = url_spec.scheme
+        shared_loc = url_spec.netloc
+        shared_path = url_spec.path
+
+        if storage_type in SharedStorageFactory._storages:
+            return SharedStorageFactory._storages[storage_type]
+
+        from cortx.utils.shared_storage import shared_storage_collection
+        agents = inspect.getmembers(shared_storage_collection, inspect.isclass)
+        for name, cls in agents:
+            if name.endswith('Storage'):
+                if storage_type == cls.name:
+                    storage_agent = cls(shared_path)
+                    SharedStorageFactory._storages[storage_type] = storage_agent
+                    return storage_agent
+
+        raise SharedStorageError(errors.ERR_INVALID_SERVICE_NAME, \
+            "Invalid service name %s.", storage_type)

Utils/shared_storage/src/shared_storage_collection.py

@@ -0,0 +1,34 @@
+# CORTX Python common library.
+# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+
+import errno
+
+from cortx.utils.shared_storage import SharedStorageAgent
+from cortx.utils.shared_storage import SharedStorageError
+
+class GlusterSharedStorage(SharedStorageAgent):
+
+    """ GlusterFS based shared storage implementation """
+
+    name = 'glusterfs'
+
+    def __init__(self, shared_path: str  = ''):
+        """ Construct an object for GlusterSharedStorage class """
+        self.shared_path = shared_path
+
+    def _fetch_path(self):
+        """ fetch path from confstore """
+        shared_path = self.shared_path
+        return shared_path

Utils/shared_storage/test/__init__.py

@@ -0,0 +1,14 @@
+# CORTX-Py-Utils: CORTX Python common library.
+# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.

Utils/shared_storage/test/test_shared_storage.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+# CORTX Python common library.
+# Copyright (c) 2021 Seagate Technology LLC and/or its Affiliates
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+# For any questions about this software or licensing,
+# please email opensource@seagate.com or cortx-questions@seagate.com.
+
+import os
+import unittest
+
+from cortx.utils.shared_storage import Storage
+from cortx.utils.conf_store import Conf
+
+class TestSharedStorage(unittest.TestCase):
+
+    """ Unit test class to test shared storage. """
+
+    @classmethod
+    def setUpClass(cls):
+        """Register the test message_type."""
+        config_file = 'json:///etc/cortx/cortx.conf'
+        Conf.load('cotrx_config', config_file, skip_reload=True)
+        cls.local_path = Conf.get('cotrx_config', 'support>local_path')
+        os.makedirs(cls.local_path, exist_ok=True)
+
+    def test_shared_path_read_access(self):
+        """ test if shared storage path exists and is readable """
+        shared_path = Storage.get_path()
+        if not shared_path:
+            shared_path=TestSharedStorage.local_path
+        self.assertTrue(os.access(shared_path, os.R_OK))
+
+    def test_shared_path_write_access(self):
+        """ test if shared storage path exists and is writable """
+        shared_path = Storage.get_path()
+        if not shared_path:
+            shared_path=TestSharedStorage.local_path
+        self.assertTrue(os.access(shared_path, os.W_OK))
+
+
+
+if __name__ == '__main__':
+    unittest.main()