Pilot 5920: merge patch fix for 3.0.16 (#182)

* update python version to 3.10 for windows * testing * testing * testing * testing * testing * testing * finish testing. uncomment all testing * test pipeline in patch branch * test pipeline in patch branch * test pipeline in patch branch * test pipeline in patch branch * test pipeline in patch branch * pipeline test finished. ready for merge * update download logic to add integrity check * update download logic to add integrity check * update comments * fixup the error when name folders or shared folders do not exist when moving * fixup the error when name folders or shared folders do not exist when moving (#167) * bumpup version * fixup move command with filename * fixup the folder download issue * test on windows binary * test on windows binary * test on windows binary * test on windows binary * test on windows binary * update windows pipeline * renaming the cache of windows system * update poetry package * build poetry.lock in linux * test on windows pipeline * undo pipeline comment and increase timeout from 10s to 30s * fixup help page when n there is no user config * add new update mesaage * add new envvar apikey enpdpoint * fixup file move error * use get function to retreive access token * update pipeline * allow build binary in feature branch temporarily * allow build binary in feature branch temporarily * allow build binary in feature branch temporarily * remove temporary build * update patch version * fixup error when refresh token expired * merged with move command fix * add time check * increate timeout of duplication check and batch get geid api to 60s * update logger for debuging * update logger for debuging * merge * remove unused code --------- Co-authored-by: Zhiren Zhan <zzhan@indocsystems.com>
PilotDataPlatform · Oct 28, 2024 · de76a23 · de76a23 · github-actions · Oct 28, 2024
1 parent c4e5052
commit de76a23
Show file tree

Hide file tree

Showing 9 changed files with 37 additions and 16 deletions.
diff --git a/app/commands/file.py b/app/commands/file.py
@@ -25,6 +25,7 @@
 from app.services.file_manager.file_upload.file_upload import resume_upload
 from app.services.file_manager.file_upload.file_upload import simple_upload
 from app.services.file_manager.file_upload.upload_validator import UploadEventValidator
+from app.services.logger_services.debugging_log import debug_logger
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import SrvErrorHandler
 from app.services.output_manager.error_handler import customized_error_msg
@@ -190,6 +191,9 @@ def file_put(**kwargs):  # noqa: C901
     # the loop will read all input path(folder or files)
     # and process them one by one
     for f in files:
+        import time
+
+        start_time = time.time()
         # so this function will always return the furthest folder node as current_folder_node+parent_folder_id
         current_folder_node, parent_folder, create_folder_flag, target_folder = assemble_path(
             f,
@@ -198,6 +202,7 @@ def file_put(**kwargs):  # noqa: C901
             folder_type,
             zone,
         )
+        debug_logger.debug(f'assemble_path: {time.time() - start_time}')
 
         upload_event = {
             'project_code': project_code,

diff --git a/app/configs/app_config.py b/app/configs/app_config.py
@@ -21,7 +21,7 @@ class Env:
         pipeline_straight_upload = f'{ConfigClass.project}cli_upload'
         default_upload_message = f'{ConfigClass.project}cli straight uploaded'
         session_duration = 3600.0
-        upload_batch_size = 100
+        upload_batch_size = ConfigClass.upload_batch_size
         core_zone = 'core'
         green_zone = 'greenroom'
         core_bucket_prefix = 'core'

diff --git a/app/configs/config.py b/app/configs/config.py
@@ -36,6 +36,8 @@ class Settings(BaseSettings):
     # for dev testing, please set the version to `api-key/v1`
     apikey_endpoint: str = 'api-key'
 
+    upload_batch_size: int = 100
+
     def __init__(self, **data):
         super().__init__(**data)
 

diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py
@@ -23,6 +23,7 @@
 from app.services.file_manager.file_upload.models import ItemStatus
 from app.services.file_manager.file_upload.models import UploadType
 from app.services.file_manager.file_upload.upload_client import UploadClient
+from app.services.logger_services.debugging_log import debug_logger
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import SrvErrorHandler
 from app.services.output_manager.error_handler import customized_error_msg
@@ -73,7 +74,9 @@ def assemble_path(
     current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1]
     # set name folder as first parent folder
     root_folder = folder_type.get_prefix_by_type() + target_folder.split('/')[0]
+    start_time = time.time()
     parent_folder = search_item(project_code, zone, root_folder).get('result', {})
+    debug_logger.debug(f'Elapsed time: {time.time() - start_time:.2f}s')
 
     # if f input is a file then current_folder_node is target_folder
     # otherwise it is target_folder + f input name
@@ -85,9 +88,12 @@ def assemble_path(
 
     if len(current_file_path.split('/')) > 2:
         sub_path = target_folder.split('/')
-        for index in range(len(sub_path) - 1):
+        # always assume <root>/<name/shared folder> exists
+        for index in range(1, len(sub_path) - 1):
             folder_path = '/'.join(sub_path[0 : 2 + index])
+            start_time = time.time()
             res = search_item(project_code, zone, folder_path)
+            debug_logger.debug(f'Elapsed time: {time.time() - start_time:.2f}s')
 
             # find the longest existing folder as parent folder
             # if user input a path that need to create some folders
@@ -187,8 +193,11 @@ def simple_upload(  # noqa: C901
     else:
         mhandler.SrvOutPutHandler.file_duplication_check()
         duplicated_file = []
+        debug_logger.debug(f'upload batch size: {AppConfig.Env.upload_batch_size}')
         for file_batchs in batch_generator(file_objects, batch_size=AppConfig.Env.upload_batch_size):
+            start_time = time.time()
             non_duplicates, duplicate_path = upload_client.check_upload_duplication(file_batchs)
+            debug_logger.debug(f'Check duplication time: {time.time() - start_time:.2f}s')
             non_duplicate_file_objects.extend(non_duplicates)
             duplicated_file.extend(duplicate_path)
 

diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py
@@ -61,7 +61,7 @@ def __init__(
         source_id: str = '',
         attributes: dict = None,
     ):
-        super().__init__('')
+        super().__init__('', timeout=60)
 
         self.user = UserConfig()
         self.operator = self.user.username

diff --git a/app/services/logger_services/debugging_log.py b/app/services/logger_services/debugging_log.py
@@ -0,0 +1,10 @@
+# Copyright (C) 2022-2024 Indoc Systems
+#
+# Contact Indoc Systems for any questions regarding the use of this source code.
+
+from logging import getLogger
+
+from app.configs.config import ConfigClass
+
+debug_logger = getLogger()
+debug_logger.setLevel(ConfigClass.log_level)
diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py
@@ -5,6 +5,7 @@
 import os
 import re
 import shutil
+import time
 from typing import Any
 from typing import Dict
 from typing import List
@@ -19,6 +20,7 @@
 from app.models.item import ItemStatus
 from app.models.item import ItemType
 from app.services.clients.base_auth_client import BaseAuthClient
+from app.services.logger_services.debugging_log import debug_logger
 from app.services.output_manager.error_handler import ECustomizedError
 from app.services.output_manager.error_handler import SrvErrorHandler
 from app.services.user_authentication.decorator import require_valid_token
@@ -71,9 +73,12 @@ def get_attribute_template_by_id(template_id: str) -> Dict[str, Any]:
 @require_valid_token()
 def get_file_info_by_geid(geid: list) -> List[Dict[str, Any]]:
     payload = {'geid': geid}
-    http_client = BaseAuthClient(AppConfig.Connections.url_bff)
+    http_client = BaseAuthClient(AppConfig.Connections.url_bff, timeout=60)
     try:
+
+        start_time = time.time()
         res = http_client._post('v1/query/geid', json=payload)
+        debug_logger.debug(f'Time taken to get file info: {time.time() - start_time}')
     except HTTPStatusError as e:
         res = e.response
         SrvErrorHandler.default_handle(res.text, True)

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "app"
-version = "3.8.2"
+version = "3.8.3"
 description = "This service is designed to support pilot platform"
 authors = ["Indoc Systems"]
 

diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py
@@ -85,7 +85,7 @@ def test_assemble_path_at_exsting_folder(mocker):
         local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone
     )
     assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/test_folder_exist/file.txt'
-    assert parent_folder.get('name') == 'test_folder_exist'
+    assert parent_folder.get('name') == 'admin'
     assert create_folder_flag is False
 
 
@@ -96,16 +96,6 @@ def test_assemble_path_at_non_existing_folder(mocker):
     zone = 0
 
     node_list = [
-        {
-            'result': {
-                'id': 'test',
-                'parent_id': 'test_parent',
-                'parent_path': '',
-                'name': ItemType.NAMEFOLDER.get_prefix_by_type().strip('/'),
-                'zone': 0,
-                'type': 'folder',
-            }
-        },
         {
             'result': {
                 'id': 'test',
File	Stmts	Miss	Cover	Missing
__init__.py	0	0	100%
pilotcli.py	0	37	0%	5, 7–9, 11–17, 20–24, 26, 28–31, 33–38, 41–47, 50–52
commands
__init__.py	0	0	100%
container_registry.py	0	19	59%	16, 22–24, 31–33, 41–44, 50–53, 62–65
dataset.py	0	9	89%	20, 38–39, 47, 82–83, 92, 126, 130
entry_point.py	0	11	85%	41, 47, 75, 98–100, 102–106
file.py	0	39	86%	47, 145–148, 152, 157, 160–162, 220, 304–311, 313, 315, 329–335, 337, 366, 368, 372, 418, 421, 436–438, 444–445
folder.py	0	1	96%	23
project.py	0	2	95%	18, 61
user.py	0	9	82%	27, 44–45, 53–54, 62–63, 84–85
configs
__init__.py	0	0	100%
app_config.py	0	0	100%
config.py	0	0	100%
user_config.py	0	21	83%	35, 70, 79, 90, 117, 130, 140, 146, 150, 154, 158, 162, 166, 170, 174, 178, 182, 186, 190, 194, 198
utils.py	0	35	36%	13–14, 16–19, 22–23, 25, 27, 31–32, 34, 36–37, 40, 43, 45–47, 49–51, 54–57, 59, 67, 69–72, 74–75
models
__init__.py	0	0	100%
enums.py	0	0	100%
item.py	0	0	100%
service_meta_class.py	0	1	80%	9
singleton.py	0	0	100%
upload_form.py	0	4	33%	28, 40–42
resources
custom_error.py	0	0	100%
custom_help.py	0	0	100%
services
__init__.py	0	0	100%
services/clients
__init__.py	0	0	100%
base_auth_client.py	0	1	96%	53
base_client.py	0	2	95%	52, 103
services/container_registry_manager
container_registry_manager.py	0	108	17%	19–20, 23–26, 29–33, 36–44, 48–60, 62–64, 68–82, 84–86, 90–99, 101–103, 107–110, 125–136, 140–143, 147–162, 164, 166–169
services/crypto
__init__.py	0	0	100%
crypto.py	0	19	40%	35, 43–47, 49, 59–61, 69–75, 77, 79
services/dataset_manager
dataset_detail.py	0	22	71%	38, 59, 66–71, 73–75, 77–86, 88
dataset_download.py	0	22	82%	53, 63–65, 74–76, 92, 94–95, 97–98, 100, 102–104, 112–115, 149, 164
dataset_list.py	0	8	81%	36–41, 43, 52
model.py	0	0	100%
services/file_manager
__init__.py	0	0	100%
file_list.py	0	12	85%	60–66, 86–88, 100, 102
file_manifests.py	0	80	30%	18–23, 41–45, 49–55, 57, 61–66, 68, 72–73, 77–81, 83, 86, 88–89, 91–92, 94–97, 102–108, 113–118, 121–127, 129–131, 134–140, 142–144, 146–149
file_tag.py	0	39	31%	23, 27–31, 33, 36–48, 50–52, 54–55, 59–61, 64–68, 70–73, 75–76
services/file_manager/file_download
__init__.py	0	0	100%
download_client.py	0	105	55%	52–61, 79–80, 122–126, 128–130, 142–143, 148–149, 153–162, 165–170, 180–181, 212, 217, 223–233, 235–236, 239–240, 245–246, 250–255, 259–260, 263–265, 267–278, 285, 290, 305, 309–311, 313–326, 328
model.py	0	0	100%
services/file_manager/file_metadata
__init__.py	0	0	100%
file_metadata_client.py	0	3	95%	98–100
folder_client.py	0	4	91%	50, 79–81
services/file_manager/file_move
__init__.py	0	0	100%
file_move_client.py	0	37	59%	60–64, 66, 74, 79–83, 86–94, 96, 98–103, 114, 116–120, 122, 172–173
services/file_manager/file_trash
__init__.py	0	0	100%
file_trash_client.py	0	2	95%	58, 71
utils.py	0	3	92%	39, 63, 70
services/file_manager/file_upload
__init__.py	0	0	100%
exception.py	0	0	100%
file_upload.py	0	28	84%	38–45, 103–105, 117, 146–147, 151, 156, 192, 215–217, 258–259, 261–263, 337, 340, 344
models.py	0	5	91%	24, 44, 150–152
upload_client.py	0	45	76%	101–104, 144, 216, 230–244, 246, 248–252, 254–258, 260–261, 391, 409, 411, 417–418, 423–426, 428–429
upload_validator.py	0	19	57%	26–32, 35–41, 44–45, 50, 52, 54
services/logger_services
__init__.py	0	0	100%
debugging_log.py	0	0	100%
log_functions.py	0	0	100%
services/output_manager
__init__.py	0	0	100%
error_handler.py	0	0	100%
help_page.py	0	7	91%	13–17, 19, 21
message_handler.py	0	56	73%	25, 47, 52, 67–69, 74, 84, 89, 102, 107, 114, 119, 124, 128, 144, 179, 189, 198, 216, 238, 282–293, 304–309, 311–317, 328, 332–333, 337–338, 340–341, 345, 349, 353
services/project_manager
__init__.py	0	0	100%
project.py	0	6	84%	40, 46–47, 49–51
services/user_authentication
__init__.py	0	0	100%
decorator.py	0	2	93%	27, 30
token_manager.py	0	9	87%	27, 46–50, 77–78, 95
user_login_logout.py	0	16	84%	29–30, 41, 126, 130–136, 138–140, 144–145
utils
__init__.py	0	0	100%
aggregated.py	0	27	83%	56, 66–68, 82–84, 116, 130–131, 144, 168–175, 177–178, 206, 214, 231–232, 240, 263
TOTAL	3598	875	75%