diff --git a/app/commands/file.py b/app/commands/file.py index 68029b1d..ab071abf 100644 --- a/app/commands/file.py +++ b/app/commands/file.py @@ -25,6 +25,7 @@ from app.services.file_manager.file_upload.file_upload import resume_upload from app.services.file_manager.file_upload.file_upload import simple_upload from app.services.file_manager.file_upload.upload_validator import UploadEventValidator +from app.services.logger_services.debugging_log import debug_logger from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.output_manager.error_handler import customized_error_msg @@ -190,6 +191,9 @@ def file_put(**kwargs): # noqa: C901 # the loop will read all input path(folder or files) # and process them one by one for f in files: + import time + + start_time = time.time() # so this function will always return the furthest folder node as current_folder_node+parent_folder_id current_folder_node, parent_folder, create_folder_flag, target_folder = assemble_path( f, @@ -198,6 +202,7 @@ def file_put(**kwargs): # noqa: C901 folder_type, zone, ) + debug_logger.debug(f'assemble_path: {time.time() - start_time}') upload_event = { 'project_code': project_code, diff --git a/app/configs/app_config.py b/app/configs/app_config.py index d666a475..643438c8 100644 --- a/app/configs/app_config.py +++ b/app/configs/app_config.py @@ -21,7 +21,7 @@ class Env: pipeline_straight_upload = f'{ConfigClass.project}cli_upload' default_upload_message = f'{ConfigClass.project}cli straight uploaded' session_duration = 3600.0 - upload_batch_size = 100 + upload_batch_size = ConfigClass.upload_batch_size core_zone = 'core' green_zone = 'greenroom' core_bucket_prefix = 'core' diff --git a/app/configs/config.py b/app/configs/config.py index 000a5417..8690be1d 100644 --- a/app/configs/config.py +++ b/app/configs/config.py @@ -36,6 +36,8 @@ class Settings(BaseSettings): # for dev testing, please set the version to `api-key/v1` apikey_endpoint: str = 'api-key' + upload_batch_size: int = 100 + def __init__(self, **data): super().__init__(**data) diff --git a/app/services/file_manager/file_upload/file_upload.py b/app/services/file_manager/file_upload/file_upload.py index 981b2712..230eac46 100644 --- a/app/services/file_manager/file_upload/file_upload.py +++ b/app/services/file_manager/file_upload/file_upload.py @@ -23,6 +23,7 @@ from app.services.file_manager.file_upload.models import ItemStatus from app.services.file_manager.file_upload.models import UploadType from app.services.file_manager.file_upload.upload_client import UploadClient +from app.services.logger_services.debugging_log import debug_logger from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.output_manager.error_handler import customized_error_msg @@ -73,7 +74,9 @@ def assemble_path( current_file_path = target_folder + '/' + f.rstrip('/').split('/')[-1] # set name folder as first parent folder root_folder = folder_type.get_prefix_by_type() + target_folder.split('/')[0] + start_time = time.time() parent_folder = search_item(project_code, zone, root_folder).get('result', {}) + debug_logger.debug(f'Elapsed time: {time.time() - start_time:.2f}s') # if f input is a file then current_folder_node is target_folder # otherwise it is target_folder + f input name @@ -85,9 +88,12 @@ def assemble_path( if len(current_file_path.split('/')) > 2: sub_path = target_folder.split('/') - for index in range(len(sub_path) - 1): + # always assume / exists + for index in range(1, len(sub_path) - 1): folder_path = '/'.join(sub_path[0 : 2 + index]) + start_time = time.time() res = search_item(project_code, zone, folder_path) + debug_logger.debug(f'Elapsed time: {time.time() - start_time:.2f}s') # find the longest existing folder as parent folder # if user input a path that need to create some folders @@ -187,8 +193,11 @@ def simple_upload( # noqa: C901 else: mhandler.SrvOutPutHandler.file_duplication_check() duplicated_file = [] + debug_logger.debug(f'upload batch size: {AppConfig.Env.upload_batch_size}') for file_batchs in batch_generator(file_objects, batch_size=AppConfig.Env.upload_batch_size): + start_time = time.time() non_duplicates, duplicate_path = upload_client.check_upload_duplication(file_batchs) + debug_logger.debug(f'Check duplication time: {time.time() - start_time:.2f}s') non_duplicate_file_objects.extend(non_duplicates) duplicated_file.extend(duplicate_path) diff --git a/app/services/file_manager/file_upload/upload_client.py b/app/services/file_manager/file_upload/upload_client.py index 5d6151ce..16481184 100644 --- a/app/services/file_manager/file_upload/upload_client.py +++ b/app/services/file_manager/file_upload/upload_client.py @@ -61,7 +61,7 @@ def __init__( source_id: str = '', attributes: dict = None, ): - super().__init__('') + super().__init__('', timeout=60) self.user = UserConfig() self.operator = self.user.username diff --git a/app/services/logger_services/debugging_log.py b/app/services/logger_services/debugging_log.py new file mode 100644 index 00000000..74920e37 --- /dev/null +++ b/app/services/logger_services/debugging_log.py @@ -0,0 +1,10 @@ +# Copyright (C) 2022-2024 Indoc Systems +# +# Contact Indoc Systems for any questions regarding the use of this source code. + +from logging import getLogger + +from app.configs.config import ConfigClass + +debug_logger = getLogger() +debug_logger.setLevel(ConfigClass.log_level) diff --git a/app/utils/aggregated.py b/app/utils/aggregated.py index f90f9744..b6ace0f5 100644 --- a/app/utils/aggregated.py +++ b/app/utils/aggregated.py @@ -5,6 +5,7 @@ import os import re import shutil +import time from typing import Any from typing import Dict from typing import List @@ -19,6 +20,7 @@ from app.models.item import ItemStatus from app.models.item import ItemType from app.services.clients.base_auth_client import BaseAuthClient +from app.services.logger_services.debugging_log import debug_logger from app.services.output_manager.error_handler import ECustomizedError from app.services.output_manager.error_handler import SrvErrorHandler from app.services.user_authentication.decorator import require_valid_token @@ -71,9 +73,12 @@ def get_attribute_template_by_id(template_id: str) -> Dict[str, Any]: @require_valid_token() def get_file_info_by_geid(geid: list) -> List[Dict[str, Any]]: payload = {'geid': geid} - http_client = BaseAuthClient(AppConfig.Connections.url_bff) + http_client = BaseAuthClient(AppConfig.Connections.url_bff, timeout=60) try: + + start_time = time.time() res = http_client._post('v1/query/geid', json=payload) + debug_logger.debug(f'Time taken to get file info: {time.time() - start_time}') except HTTPStatusError as e: res = e.response SrvErrorHandler.default_handle(res.text, True) diff --git a/pyproject.toml b/pyproject.toml index 835d4d9e..b1756230 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "app" -version = "3.8.2" +version = "3.8.3" description = "This service is designed to support pilot platform" authors = ["Indoc Systems"] diff --git a/tests/app/services/file_manager/file_upload/test_file_upload.py b/tests/app/services/file_manager/file_upload/test_file_upload.py index 907d51b9..9faa7cdc 100644 --- a/tests/app/services/file_manager/file_upload/test_file_upload.py +++ b/tests/app/services/file_manager/file_upload/test_file_upload.py @@ -85,7 +85,7 @@ def test_assemble_path_at_exsting_folder(mocker): local_file_path, target_folder, project_code, ItemType.NAMEFOLDER, zone ) assert current_file_path == f'{ItemType.NAMEFOLDER.get_prefix_by_type()}admin/test_folder_exist/file.txt' - assert parent_folder.get('name') == 'test_folder_exist' + assert parent_folder.get('name') == 'admin' assert create_folder_flag is False @@ -96,16 +96,6 @@ def test_assemble_path_at_non_existing_folder(mocker): zone = 0 node_list = [ - { - 'result': { - 'id': 'test', - 'parent_id': 'test_parent', - 'parent_path': '', - 'name': ItemType.NAMEFOLDER.get_prefix_by_type().strip('/'), - 'zone': 0, - 'type': 'folder', - } - }, { 'result': { 'id': 'test',