Mb/staging release 20240307 (#131)

* PILOT-3962: Port over changes from 2.7.3 to 2.7.4

* prepare the release branch

* use correct version

* bumup version

* bump up to correct version

* Hotfix20240228: hotfix to staging (#129)

* update folder name restriction to 100 char

* add error handling when uploading with wrong format of tag/attribute files

* fixup the metadata download command cannt download item metadata/tags/attributes from project folder

* bumpup version

---------

Co-authored-by: zhiren <zzhan@indocresearch.org>

* PILOT-4745: fixup the move command fails when creating a new folder under project folder (#130)

* fixup the move command fails when creating a new folder under project folder

* bumpup the version

---------

Co-authored-by: zhiren <zzhan@indocresearch.org>

* bumpup to next preminor

---------

Co-authored-by: Daniel <dfelipe@indocresearch.org>
Co-authored-by: zhiren <zzhan@indocresearch.org>
Co-authored-by: Dušan Andrić <Dusan.Andric@gmail.com>

app/commands/file.py

@@ -132,10 +132,16 @@ def file_put(**kwargs):  # noqa: C901
     output_path = kwargs.get('output_path')
 
     # load tag json file to list, and attribute file to dict
-    tag = []
-    for t_f in tag_files:
-        tag.extend(json.load(t_f))
-    attribute = json.load(attribute_file) if attribute_file else None
+    try:
+        tag = []
+        for t_f in tag_files:
+            tag.extend(json.load(t_f))
+    except Exception:
+        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TAG_FILE, True)
+    try:
+        attribute = json.load(attribute_file) if attribute_file else None
+    except Exception:
+        SrvErrorHandler.customized_handle(ECustomizedError.INVALID_TEMPLATE, True)
 
     # Check zone and upload-message
     zone = get_zone(zone) if zone else AppConfig.Env.green_zone.lower()

app/resources/custom_error.py

@@ -23,6 +23,7 @@ class Error:
             "Attribute validation failed. Please ensure mandatory attribute '%s' have value and try again."
         ),
         'INVALID_TEMPLATE': 'Attribute validation failed. Please correct JSON format and try again.',
+        'INVALID_TAG_FILE': 'Tag files validation failed. Please correct JSON format and try again.',
         'LIMIT_TAG_ERROR': 'Tag limit has been reached. A maximum of 10 tags are allowed per file.',
         'INVALID_TAG_ERROR': (
             'Invalid tag format. Tags must be between 1 and 32 characters long '
@@ -45,7 +46,7 @@ class Error:
         'INVALID_FOLDERNAME': (
             'The input folder name is not valid. Please follow the rule:\n'
             ' - cannot contains special characters.\n'
-            ' - the length should be smaller than 20 characters.'
+            ' - the length should be smaller than or equal to 100 characters.'
         ),
         'INVALID_TOKEN': 'Your login session has expired. Please try again or log in again.',
         'PERMISSION_DENIED': (

app/services/file_manager/file_metadata/file_metadata_client.py

@@ -109,7 +109,15 @@ def download_file_metadata(self) -> List[Dict[str, Any]]:
         """
 
         project_code, object_path = self.file_path.split('/', 1)
-        item_res = search_item(project_code, self.zone, object_path).get('result', {})
+        item_res = search_item(project_code, self.zone, object_path)
+        # double check if the file is in shared folder
+        if item_res.get('code') == 404:
+            item_res = search_item(project_code, self.zone, f'shared/{object_path}')
+            if item_res.get('code') == 404:
+                logger.error(f'Cannot find item {self.file_path} at {self.zone}.')
+                exit(1)
+
+        item_res = item_res.get('result', {})
         extra_info = item_res.pop('extended', {}).get('extra')
         tags = extra_info.get('tags', [])
         attributes = extra_info.get('attributes', {})

app/services/file_manager/file_move/file_move_client.py

@@ -60,6 +60,12 @@ def create_object_path_if_not_exist(self, folder_path: str) -> dict:
         """
 
         path_list = folder_path.split('/')
+        # first get the root folder to check if it is name folder
+        # or project folder
+        root_item = search_item(self.project_code, self.zone, path_list[0]).get('result')
+        if root_item.get('type') == 'project_folder':
+            path_list[0] = '/'.join([root_item.get('parent_path'), path_list[0]])
+
         # first check every folder in path exist or not
         # the loop start with index 1 since we assume cli will not
         # create any name folder or project folder

app/services/output_manager/error_handler.py

@@ -22,6 +22,7 @@ class ECustomizedError(enum.Enum):
     TEXT_TOO_LONG = 'TEXT_TOO_LONG'
     FIELD_REQUIRED = 'FIELD_REQUIRED'
     INVALID_TEMPLATE = 'INVALID_TEMPLATE'
+    INVALID_TAG_FILE = 'INVALID_TAG_FILE'
     LIMIT_TAG_ERROR = 'LIMIT_TAG_ERROR'
     INVALID_TAG_ERROR = 'INVALID_TAG_ERROR'
     RESERVED_TAG = 'RESERVED_TAG'

app/utils/aggregated.py

@@ -133,7 +133,7 @@ def get_zone(zone):
 def validate_folder_name(folder_name):
     regex = re.compile('[/:?.\\*<>|”\']')
     contain_invalid_char = regex.search(folder_name)
-    if contain_invalid_char or len(folder_name) > 20 or not folder_name:
+    if contain_invalid_char or len(folder_name) > 100 or not folder_name:
         valid = False
     else:
         valid = True

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "2.9.8"
+version = "2.10.0a0"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]
 

tests/app/commands/test_file.py

@@ -55,6 +55,35 @@ def test_file_upload_command_success_with_attribute(mocker, cli_runner):
     attribute_mock.assert_called_once()
 
 
+def test_file_upload_failed_with_invalid_tag_file(cli_runner):
+    # create invalid tag file with wrong format
+    runner = click.testing.CliRunner()
+    with runner.isolated_filesystem():
+        with open('wrong_tag.json', 'w') as f:
+            f.write('wrong_tag.json')
+
+        result = cli_runner.invoke(
+            file_put, ['--project-path', 'test', '--thread', 1, '--tag', 'wrong_tag.json', 'wrong_tag.json']
+        )
+    assert result.exit_code == 0
+    assert result.output == customized_error_msg(ECustomizedError.INVALID_TAG_FILE) + '\n'
+
+
+def test_file_upload_failed_with_invalid_attribute_file(cli_runner):
+    # create invalid attribute file with wrong format
+    runner = click.testing.CliRunner()
+    with runner.isolated_filesystem():
+        with open('wrong_attribute.json', 'w') as f:
+            f.write('wrong_attribute.json')
+
+        result = cli_runner.invoke(
+            file_put,
+            ['--project-path', 'test', '--thread', 1, '--attribute', 'wrong_attribute.json', 'wrong_attribute.json'],
+        )
+    assert result.exit_code == 0
+    assert result.output == customized_error_msg(ECustomizedError.INVALID_TEMPLATE) + '\n'
+
+
 def test_resumable_upload_command_success(mocker, cli_runner):
     mocker.patch('os.path.exists', return_value=True)
     # mock the open function

tests/app/services/file_manager/file_metadata/test_file_metadata_client.py

@@ -84,3 +84,76 @@ def test_file_metadata_client_get_detail_success_with_no_tag_attributes(mocker,
     assert item_info == item_info
     assert res_attributes == {}
     assert tags == tags
+
+
+def test_metadata_download_from_project_folder(mocker, httpx_mock):
+    item_info = {
+        'id': 'test',
+        'parent_id': 'test_parent',
+        'parent_path': 'shared/path',
+        'name': 'admin',
+        'zone': 0,
+        'status': 'ACTIVE',
+    }
+    tags = ['test']
+    attri_template_uid = 'template_uid'
+    attri_template_name = 'template_name'
+    attributes = {attri_template_uid: {'attr_1': 'value'}}
+
+    mocker.patch(
+        'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
+        return_value=decoded_token(),
+    )
+
+    search_mock = mocker.patch(
+        'app.services.file_manager.file_metadata.file_metadata_client.search_item',
+    )
+    search_mock.side_effect = [
+        {'result': {}, 'code': 404},
+        {'result': {**item_info, 'extended': {'extra': {'tags': tags, 'attributes': attributes}}}},
+    ]
+    httpx_mock.add_response(
+        url=AppConfig.Connections.url_portal + f'/v1/data/manifest/{attri_template_uid}',
+        method='GET',
+        json={'result': {'id': attri_template_uid, 'name': attri_template_name}},
+    )
+
+    mocker.patch(
+        'app.services.file_manager.file_metadata.file_metadata_client.FileMetaClient.save_file_metadata',
+        return_value=None,
+    )
+
+    file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag')
+    assert file_meta_client.project_code == 'project_code'
+    assert file_meta_client.object_path == 'object_path'
+
+    item_info, res_attributes, tags = file_meta_client.download_file_metadata()
+    assert item_info == item_info
+    assert res_attributes == {attri_template_name: attributes.get(attri_template_uid)}
+    assert tags == tags
+    assert search_mock.call_count == 2
+
+
+def test_metadata_download_fail_when_file_doesnot_exist(mocker, capfd):
+    mocker.patch(
+        'app.services.user_authentication.token_manager.SrvTokenManager.decode_access_token',
+        return_value=decoded_token(),
+    )
+
+    search_mock = mocker.patch(
+        'app.services.file_manager.file_metadata.file_metadata_client.search_item',
+    )
+    search_mock.side_effect = [{'result': {}, 'code': 404}, {'result': {}, 'code': 404}]
+
+    file_meta_client = FileMetaClient('zone', 'project_code/object_path', 'general', 'attr', 'tag')
+
+    try:
+        file_meta_client.download_file_metadata()
+    except SystemExit:
+        assert search_mock.call_count == 2
+        out, _ = capfd.readouterr()
+
+        expect = 'Cannot find item project_code/object_path at zone.\n'
+        assert out == expect
+    else:
+        AssertionError('SystemExit not raised')

tests/app/services/file_manager/file_move/test_file_move_client.py

@@ -18,6 +18,11 @@ def test_file_move_success(mocker, httpx_mock):
         return_value=decoded_token(),
     )
 
+    mocker.patch(
+        'app.services.file_manager.file_move.file_move_client.FileMoveClient.create_object_path_if_not_exist',
+        return_value=[],
+    )
+
     httpx_mock.add_response(
         url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files',
         method='PATCH',
@@ -37,6 +42,11 @@ def test_file_move_error_with_permission_denied_403(mocker, httpx_mock, capfd):
         return_value=decoded_token(),
     )
 
+    mocker.patch(
+        'app.services.file_manager.file_move.file_move_client.FileMoveClient.create_object_path_if_not_exist',
+        return_value=[],
+    )
+
     httpx_mock.add_response(
         url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files',
         method='PATCH',
@@ -60,6 +70,11 @@ def test_file_move_error_with_wrong_input_422(mocker, httpx_mock, capfd):
         return_value=decoded_token(),
     )
 
+    mocker.patch(
+        'app.services.file_manager.file_move.file_move_client.FileMoveClient.create_object_path_if_not_exist',
+        return_value=[],
+    )
+
     httpx_mock.add_response(
         url=AppConfig.Connections.url_bff + f'/v1/{project_code}/files',
         method='PATCH',

tests/app/utils/test_aggregated.py

@@ -7,6 +7,7 @@
 from app.configs.app_config import AppConfig
 from app.utils.aggregated import check_item_duplication
 from app.utils.aggregated import search_item
+from app.utils.aggregated import validate_folder_name
 from tests.conftest import decoded_token
 
 test_project_code = 'testproject'
@@ -122,3 +123,9 @@ def test_check_duplicate_fail_with_error_code(httpx_mock, mocker, capsys):
         check_item_duplication(['test_path'], 0, 'test_project_code')
     out, _ = capsys.readouterr()
     assert out.rstrip() == '{"error": "internal server error"}'
+
+
+@pytest.mark.parametrize('folder_name', ['/:?.\\*<>|”\'', ''.join(['1' for _ in range(101)])])
+def test_validate_folder_name(folder_name):
+    valid = validate_folder_name(folder_name)
+    assert valid is False