From 6e699e73cd00c892a49c42f8531010041989a2f9 Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 23 Jan 2024 16:37:22 -0500 Subject: [PATCH 1/5] add a new logic to allow cli to create non-exist folder when moving or renaming --- .../file_move/file_move_client.py | 81 +++++++++++++++++++ .../output_manager/message_handler.py | 5 ++ 2 files changed, 86 insertions(+) diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 4bd2dfb4..02ed7c22 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -2,10 +2,20 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import uuid +from sys import exit + +import click +from click import Abort + import app.services.output_manager.message_handler as message_handler from app.configs.app_config import AppConfig from app.configs.user_config import UserConfig +from app.services.output_manager.error_handler import ECustomizedError +from app.services.output_manager.error_handler import SrvErrorHandler +from app.services.output_manager.error_handler import customized_error_msg from app.utils.aggregated import resilient_session +from app.utils.aggregated import search_item class FileMoveClient: @@ -39,12 +49,83 @@ def __init__( self.user = UserConfig() + def create_object_path_if_not_exist(self, folder_path: str) -> dict: + """Create object path is not on platfrom. + + it will create every non-exist folder along path. + """ + + path_list = folder_path.split('/') + # first check every folder in path exist or not + # the loop start with index 1 since we assume cli will not + # create any name folder or project folder + check_list = [] + for index in range(1, len(path_list) - 1): + check_list.append('/'.join(path_list[: index + 1])) + if len(check_list) == 0: + return + + url = AppConfig.Connections.url_base + '/portal/v1/files/exists' + zone_int = {'greenroom': 0, 'core': 1}.get(self.zone.lower()) + headers = {'Authorization': 'Bearer ' + UserConfig().access_token} + payload = { + 'locations': check_list, + 'container_code': self.project_code, + 'container_type': 'project', + 'zone': zone_int, + } + response = resilient_session().post(url, json=payload, headers=headers) + if response.status_code != 200: + SrvErrorHandler.default_handle(response.text, True) + + # confirm if user want to create folder or not + exist_path = response.json().get('result') + not_exist_path = sorted(set(check_list) - set(exist_path)) + if not_exist_path: + try: + click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) + except Abort: + message_handler.SrvOutPutHandler.move_cancelled() + exit(1) + else: + return + + # get the current exist parent folder for reference + exist_parent = not_exist_path[0].rsplit('/', 1)[0] + exist_parent_item = search_item(self.project_code, self.zone, exist_parent).get('result') + exist_parent_id = exist_parent_item.get('id') + to_create = {'folders': [], 'parent_id': exist_parent_id} + for path in not_exist_path: + parent_path, folder_name = path.rsplit('/', 1) + current_item_id = str(uuid.uuid4()) + to_create['folders'].append( + { + 'name': folder_name, + 'parent': exist_parent_id, + 'parent_path': parent_path, + 'container_code': self.project_code, + 'container_type': 'project', + 'zone': zone_int, + 'item_id': current_item_id, + } + ) + exist_parent_id = current_item_id + + url = AppConfig.Connections.url_bff + '/v1/folders/batch' + headers = {'Authorization': 'Bearer ' + UserConfig().access_token} + response = resilient_session().post(url, json=to_create, headers=headers) + if response.status_code != 200: + SrvErrorHandler.default_handle(response.text, True) + return response.json().get('result') + def move_file(self) -> None: """ Summary: Move file. """ + self.create_object_path_if_not_exist(self.dest_item_path) + try: url = AppConfig.Connections.url_bff + f'/v1/{self.project_code}/files' payload = { diff --git a/app/services/output_manager/message_handler.py b/app/services/output_manager/message_handler.py index cdc57677..501d0dc3 100644 --- a/app/services/output_manager/message_handler.py +++ b/app/services/output_manager/message_handler.py @@ -208,6 +208,11 @@ def move_action_success(src, dest): """e.g. Move action succeed.""" return logger.succeed(f'Successfully moved {src} to {dest}') + @staticmethod + def move_cancelled(): + """e.g. Move action cancelled.""" + return logger.warning('Move cancelled.') + @staticmethod def move_action_failed(src, dest, error): """e.g. Move action failed.""" From 619a537e37037ce1e62f23de484ecdd1961d88de Mon Sep 17 00:00:00 2001 From: zhiren Date: Tue, 23 Jan 2024 16:41:52 -0500 Subject: [PATCH 2/5] add a new boolean option to skip prompt confirmation in pipeline --- app/commands/file.py | 12 +++++++++++- app/resources/custom_help.py | 1 + .../file_manager/file_move/file_move_client.py | 5 ++++- app/services/output_manager/help_page.py | 1 + 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/commands/file.py b/app/commands/file.py index 2e821aa8..f38b0f13 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -521,6 +521,15 @@ def file_metadata_download(**kwargs): help=file_help.file_help_page(file_help.FileHELP.FILE_MOVE_Z), show_default=False, ) +@click.option( + '-y', + '--yes', + default=False, + required=False, + is_flag=True, + help=file_help.file_help_page(file_help.FileHELP.FILE_MOVE_Y), + show_default=True, +) @require_valid_token() @doc(file_help.file_help_page(file_help.FileHELP.FILE_MOVE)) def file_move(**kwargs): @@ -528,9 +537,10 @@ def file_move(**kwargs): src_item_path = kwargs.get('src_item_path') dest_item_path = kwargs.get('dest_item_path') zone = kwargs.get('zone') + skip_confirm = kwargs.get('yes') zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower() - file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path) + file_meta_client = FileMoveClient(zone, project_code, src_item_path, dest_item_path, skip_confirm=skip_confirm) file_meta_client.move_file() message_handler.SrvOutPutHandler.move_action_success(src_item_path, dest_item_path) diff --git a/app/resources/custom_help.py b/app/resources/custom_help.py index ba86ddd9..75e77563 100644 --- a/app/resources/custom_help.py +++ b/app/resources/custom_help.py @@ -62,6 +62,7 @@ class HelpPage: 'FILE_META_T': 'The location of tag metadata file', 'FILE_MOVE': 'Move/Rename files/folders to a given Project path.', 'FILE_MOVE_Z': 'Target Zone (i.e., core/greenroom).', + 'FILE_MOVE_Y': 'Skip the prompt confirmation and create non-existing folders.', }, 'config': { 'SET_CONFIG': 'Chose config file and set for cli.', diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 02ed7c22..1c4e46a1 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -31,6 +31,7 @@ def __init__( project_code: str, src_item_path: str, dest_item_path: str, + skip_confirm: bool = False, ) -> None: """ Summary: @@ -46,6 +47,7 @@ def __init__( self.project_code = project_code self.src_item_path = src_item_path self.dest_item_path = dest_item_path + self.skip_confirm = skip_confirm self.user = UserConfig() @@ -83,7 +85,8 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict: not_exist_path = sorted(set(check_list) - set(exist_path)) if not_exist_path: try: - click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) + if not self.skip_confirm: + click.confirm(customized_error_msg(ECustomizedError.CREATE_FOLDER_IF_NOT_EXIST), abort=True) except Abort: message_handler.SrvOutPutHandler.move_cancelled() exit(1) diff --git a/app/services/output_manager/help_page.py b/app/services/output_manager/help_page.py index bfabf8e2..32cb60ac 100644 --- a/app/services/output_manager/help_page.py +++ b/app/services/output_manager/help_page.py @@ -81,6 +81,7 @@ class FileHELP(enum.Enum): FILE_MOVE = 'FILE_MOVE' FILE_MOVE_Z = 'FILE_MOVE_Z' + FILE_MOVE_Y = 'FILE_MOVE_Y' def file_help_page(FileHELP: FileHELP): From 1f45643eb74bb3856e09c35993af667eb09b4053 Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 24 Jan 2024 16:10:50 -0500 Subject: [PATCH 3/5] add the test cases for file move client --- .../file_move/file_move_client.py | 22 +++------ app/utils/aggregated.py | 28 ++++++++++++ .../file_move/test_file_move_client.py | 45 +++++++++++++++++++ 3 files changed, 79 insertions(+), 16 deletions(-) diff --git a/app/services/file_manager/file_move/file_move_client.py b/app/services/file_manager/file_move/file_move_client.py index 1c4e46a1..41833891 100644 --- a/app/services/file_manager/file_move/file_move_client.py +++ b/app/services/file_manager/file_move/file_move_client.py @@ -14,6 +14,8 @@ from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.output_manager.error_handler import customized_error_msg +from app.services.user_authentication.decorator import require_valid_token +from app.utils.aggregated import check_item_duplication from app.utils.aggregated import resilient_session from app.utils.aggregated import search_item @@ -43,7 +45,7 @@ def __init__( dest_item_path (str): destination item path. """ - self.zone = zone + self.zone = {'greenroom': 0, 'core': 1}.get(zone) self.project_code = project_code self.src_item_path = src_item_path self.dest_item_path = dest_item_path @@ -67,21 +69,8 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict: if len(check_list) == 0: return - url = AppConfig.Connections.url_base + '/portal/v1/files/exists' - zone_int = {'greenroom': 0, 'core': 1}.get(self.zone.lower()) - headers = {'Authorization': 'Bearer ' + UserConfig().access_token} - payload = { - 'locations': check_list, - 'container_code': self.project_code, - 'container_type': 'project', - 'zone': zone_int, - } - response = resilient_session().post(url, json=payload, headers=headers) - if response.status_code != 200: - SrvErrorHandler.default_handle(response.text, True) - # confirm if user want to create folder or not - exist_path = response.json().get('result') + exist_path = check_item_duplication(check_list, self.zone, self.project_code) not_exist_path = sorted(set(check_list) - set(exist_path)) if not_exist_path: try: @@ -108,7 +97,7 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict: 'parent_path': parent_path, 'container_code': self.project_code, 'container_type': 'project', - 'zone': zone_int, + 'zone': self.zone, 'item_id': current_item_id, } ) @@ -121,6 +110,7 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict: SrvErrorHandler.default_handle(response.text, True) return response.json().get('result') + @require_valid_token() def move_file(self) -> None: """ Summary: diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index 965f682b..95ac1a97 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -74,6 +74,34 @@ def get_file_info_by_geid(geid: list): return res.json()['result'] +@require_valid_token() +def check_item_duplication(item_list: List[str], zone: int, project_code: str) -> List[str]: + ''' + Summary: + Check if the item already exists in the project in batch. + Parameters: + - item_list: list of item path to check + - zone: zone of the project + - project_code: project code + Returns: + - list of item path that already exists in the project + ''' + + url = AppConfig.Connections.url_base + '/portal/v1/files/exists' + headers = {'Authorization': 'Bearer ' + UserConfig().access_token} + payload = { + 'locations': item_list, + 'container_code': project_code, + 'container_type': 'project', + 'zone': zone, + } + response = resilient_session().post(url, json=payload, headers=headers) + if response.status_code != 200: + SrvErrorHandler.default_handle(response.text, True) + + return response.json().get('result') + + def fit_terminal_width(string_to_format): string_to_format = string_to_format.rsplit('...') current_len = 0 diff --git a/tests/app/services/file_manager/file_move/test_file_move_client.py b/tests/app/services/file_manager/file_move/test_file_move_client.py index b6c52b1d..e22776e6 100644 --- a/tests/app/services/file_manager/file_move/test_file_move_client.py +++ b/tests/app/services/file_manager/file_move/test_file_move_client.py @@ -2,6 +2,8 @@ # # Contact Indoc Systems for any questions regarding the use of this source code. +import pytest + from app.configs.app_config import AppConfig from app.services.file_manager.file_move.file_move_client import FileMoveClient from tests.conftest import decoded_token @@ -71,3 +73,46 @@ def test_file_move_error_with_wrong_input_422(mocker, httpx_mock, capfd): except SystemExit: out, _ = capfd.readouterr() assert out == 'Failed to move src_item_path to dest_item_path: \nerror_msg\n' + + +@pytest.mark.parametrize('skip_confirmation', [True, False]) +def test_move_file_dest_parent_not_exist_success(mocker, httpx_mock, skip_confirmation): + project_code = 'test_code' + item_info = {'result': {'id': 'test_id', 'name': 'test_name'}} + + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + # mock duplicated item + mocker.patch( + 'app.services.file_manager.file_move.file_move_client.check_item_duplication', + return_value=[], + ) + mocker.patch( + 'app.services.file_manager.file_move.file_move_client.search_item', + return_value={'result': {'id': 'test_id', 'name': 'test_name'}}, + ) + click_mocker = mocker.patch('app.services.file_manager.file_move.file_move_client.click.confirm', return_value=None) + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + '/v1/folders/batch', + method='POST', + json={'result': []}, + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files', + method='PATCH', + json={'result': item_info}, + ) + + file_move_client = FileMoveClient( + 'zone', project_code, 'src_item_path', 'dest_item_path/test_folder/test_file', skip_confirm=skip_confirmation + ) + res = file_move_client.move_file() + assert res == item_info + if not skip_confirmation: + click_mocker.assert_called_once() + else: + click_mocker.assert_not_called() From 8d387ab5fd9475cc09c3c04e22d8c818e43333fd Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 24 Jan 2024 16:17:51 -0500 Subject: [PATCH 4/5] add a test case to check the error handling of an aggregated function --- tests/app/utils/test_aggregated.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/app/utils/test_aggregated.py b/tests/app/utils/test_aggregated.py index d7a8aa4c..34a02646 100644 --- a/tests/app/utils/test_aggregated.py +++ b/tests/app/utils/test_aggregated.py @@ -4,7 +4,10 @@ import pytest +from app.configs.app_config import AppConfig +from app.utils.aggregated import check_item_duplication from app.utils.aggregated import search_item +from tests.conftest import decoded_token test_project_code = 'testproject' @@ -100,3 +103,22 @@ def test_search_file_error_handling_with_401(requests_mock, mocker, capsys): search_item(test_project_code, 'zone', 'folder_relative_path', 'project') out, _ = capsys.readouterr() assert out.rstrip() == 'Your login session has expired. Please try again or log in again.' + + +def test_check_duplicate_fail_with_error_code(httpx_mock, mocker, capsys): + mocker.patch( + 'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token', + return_value=decoded_token(), + ) + + httpx_mock.add_response( + url=AppConfig.Connections.url_base + '/portal/v1/files/exists', + method='POST', + json={'error': 'internal server error'}, + status_code=500, + ) + + with pytest.raises(SystemExit): + check_item_duplication(['test_path'], 0, 'test_project_code') + out, _ = capsys.readouterr() + assert out.rstrip() == '{"error": "internal server error"}' From d6a89dcbfc550859588fc5e5b575880acc19cf4d Mon Sep 17 00:00:00 2001 From: zhiren Date: Wed, 24 Jan 2024 16:47:37 -0500 Subject: [PATCH 5/5] bumpup versions --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 606b776e..aba8fe43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "2.9.7" +version = "2.9.8" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"]