From fcb5f6b1e88f319dbcf46f4ed5e745a665f5e536 Mon Sep 17 00:00:00 2001 From: nitish-egov <137176807+nitish-egov@users.noreply.github.com> Date: Thu, 5 Dec 2024 09:43:47 +0530 Subject: [PATCH] added changes for performance testing (#1236) * added changes for performance testing * microplan save topic changes (#1231) * Update microplanUtils.ts * Update index.ts * Update campaignApis.ts (#1232) * Update campaignApis.ts * Update campaignApis.ts * Update campaignApis.ts * Update campaignValidators.ts * Revert boundaryProject Mapping * Cleaned up data configs (#1234) * Update index.ts * Update campaignUtils.ts * try catch handling * Update health-services/project-factory/src/server/service/dataManageService.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --------- Co-authored-by: ashish-egov <137176738+ashish-egov@users.noreply.github.com> Co-authored-by: ansh-egov <137172017+ansh-egov@users.noreply.github.com> Co-authored-by: Jagankumar <53823168+jagankumar-egov@users.noreply.github.com> Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .../src/server/api/genericApis.ts | 195 ++++++++++-------- .../src/server/config/index.ts | 1 + .../src/server/service/dataManageService.ts | 15 +- .../src/server/utils/campaignMappingUtils.ts | 109 +++++++--- .../server/validators/campaignValidators.ts | 4 +- 5 files changed, 195 insertions(+), 129 deletions(-) diff --git a/health-services/project-factory/src/server/api/genericApis.ts b/health-services/project-factory/src/server/api/genericApis.ts index 5a633000d63..2ae5f0f7491 100644 --- a/health-services/project-factory/src/server/api/genericApis.ts +++ b/health-services/project-factory/src/server/api/genericApis.ts @@ -136,10 +136,10 @@ function getRawCellValue(cell: any) { return cell.value.richText.map((rt: any) => rt.text).join(''); } else if ('hyperlink' in cell.value) { - if(cell?.value?.text?.richText?.length > 0){ + if (cell?.value?.text?.richText?.length > 0) { return cell.value.text.richText.map((t: any) => t.text).join(''); } - else{ + else { return cell.value.text; } } @@ -147,7 +147,7 @@ function getRawCellValue(cell: any) { // Get the result of the formula return cell.value.result; } - else if('sharedFormula' in cell.value){ + else if ('sharedFormula' in cell.value) { // Get the result of the shared formula return cell.value.result; } @@ -844,65 +844,65 @@ async function getBoundarySheetData( } } -async function getConfigurableColumnHeadersBasedOnCampaignTypeForBoundaryManagement(request:any, localizationMap?: { [key: string]: string }) { -try { - const mdmsResponse = await callMdmsTypeSchema( - request, - request?.query?.tenantId || request?.body?.ResourceDetails?.tenantId, - false, - request?.query?.type || request?.body?.ResourceDetails?.type, - "all" - ); - if (!mdmsResponse || mdmsResponse?.columns.length === 0) { - logger.error( - `Campaign Type all has not any columns configured in schema` - ); - throwError( - "COMMON", - 400, - "SCHEMA_ERROR", - `Campaign Type all has not any columns configured in schema` +async function getConfigurableColumnHeadersBasedOnCampaignTypeForBoundaryManagement(request: any, localizationMap?: { [key: string]: string }) { + try { + const mdmsResponse = await callMdmsTypeSchema( + request, + request?.query?.tenantId || request?.body?.ResourceDetails?.tenantId, + false, + request?.query?.type || request?.body?.ResourceDetails?.type, + "all" ); - } - // Extract columns from the response - const columnsForGivenCampaignId = mdmsResponse?.columns; + if (!mdmsResponse || mdmsResponse?.columns.length === 0) { + logger.error( + `Campaign Type all has not any columns configured in schema` + ); + throwError( + "COMMON", + 400, + "SCHEMA_ERROR", + `Campaign Type all has not any columns configured in schema` + ); + } + // Extract columns from the response + const columnsForGivenCampaignId = mdmsResponse?.columns; - // Get localized headers based on the column names - const headerColumnsAfterHierarchy = getLocalizedHeaders( - columnsForGivenCampaignId, - localizationMap - ); - if ( - !headerColumnsAfterHierarchy.includes( - getLocalizedName(config.boundary.boundaryCode, localizationMap) - ) - ) { - logger.error( - `Column Headers of generated Boundary Template does not have ${getLocalizedName( - config.boundary.boundaryCode, - localizationMap - )} column` + // Get localized headers based on the column names + const headerColumnsAfterHierarchy = getLocalizedHeaders( + columnsForGivenCampaignId, + localizationMap ); + if ( + !headerColumnsAfterHierarchy.includes( + getLocalizedName(config.boundary.boundaryCode, localizationMap) + ) + ) { + logger.error( + `Column Headers of generated Boundary Template does not have ${getLocalizedName( + config.boundary.boundaryCode, + localizationMap + )} column` + ); + throwError( + "COMMON", + 400, + "VALIDATION_ERROR", + `Column Headers of generated Boundary Template does not have ${getLocalizedName( + config.boundary.boundaryCode, + localizationMap + )} column` + ); + } + return headerColumnsAfterHierarchy; + } catch (error: any) { + console.log(error); throwError( - "COMMON", + "FILE", 400, - "VALIDATION_ERROR", - `Column Headers of generated Boundary Template does not have ${getLocalizedName( - config.boundary.boundaryCode, - localizationMap - )} column` + "FETCHING_COLUMN_ERROR", + "Error fetching column Headers From Schema (either boundary code column not found or given Campaign Type not found in schema) Check logs" ); } - return headerColumnsAfterHierarchy; -} catch (error: any) { - console.log(error); - throwError( - "FILE", - 400, - "FETCHING_COLUMN_ERROR", - "Error fetching column Headers From Schema (either boundary code column not found or given Campaign Type not found in schema) Check logs" - ); -} } async function createStaff(resouceBody: any) { // Create staff @@ -985,44 +985,63 @@ async function createProjectFacility(resouceBody: any) { } // Helper function to create staff -const createStaffHelper = (resourceId: any, projectId: any, resouceBody: any, tenantId: any, startDate: any, endDate: any) => { - const ProjectStaff = { - tenantId: tenantId.split(".")?.[0], - projectId, - userId: resourceId, - startDate, - endDate, - }; - const newResourceBody = { ...resouceBody, ProjectStaff }; - return createStaff(newResourceBody); +const createProjectStaffHelper = (resourceId: any, projectId: any, resouceBody: any, tenantId: any, startDate: any, endDate: any) => { + try { + const ProjectStaff = { + tenantId: tenantId.split(".")?.[0], + projectId, + userId: resourceId, + startDate, + endDate, + }; + const newResourceBody = { ...resouceBody, ProjectStaff }; + return createStaff(newResourceBody); + } catch (error) { + // Log the error if the API call fails + logger.error(`Failed to create project staff for staffId ${resourceId}:`, error); + throw error; // Rethrow the error to propagate it + } }; // Helper function to create project resource const createProjectResourceHelper = (resourceId: any, projectId: any, resouceBody: any, tenantId: any, startDate: any, endDate: any) => { - const ProjectResource = { - tenantId: tenantId.split(".")?.[0], - projectId, - resource: { - productVariantId: resourceId, - type: "DRUG", - isBaseUnitVariant: false, - }, - startDate, - endDate, - }; - const newResourceBody = { ...resouceBody, ProjectResource }; - return createProjectResource(newResourceBody); + try { + const ProjectResource = { + tenantId: tenantId.split(".")?.[0], + projectId, + resource: { + productVariantId: resourceId, + type: "DRUG", + isBaseUnitVariant: false, + }, + startDate, + endDate, + }; + const newResourceBody = { ...resouceBody, ProjectResource }; + return createProjectResource(newResourceBody); + } + catch (error) { + // Log the error if the API call fails + logger.error(`Failed to create project resource for resourceId ${resourceId}:`, error); + throw error; // Rethrow the error to propagate it + } }; // Helper function to create project facility -const createProjectFacilityHelper = (resourceId: any, projectId: any, resouceBody: any, tenantId: any) => { - const ProjectFacility = { - tenantId: tenantId.split(".")?.[0], - projectId, - facilityId: resourceId, - }; - const newResourceBody = { ...resouceBody, ProjectFacility }; - return createProjectFacility(newResourceBody); +const createProjectFacilityHelper = (resourceId: any, projectId: any, resouceBody: any, tenantId: any, startDate: any, endDate: any) => { + try { + const ProjectFacility = { + tenantId: tenantId.split(".")?.[0], + projectId, + facilityId: resourceId, + }; + const newResourceBody = { ...resouceBody, ProjectFacility }; + return createProjectFacility(newResourceBody); + } catch (error) { + // Log the error if the API call fails + logger.error(`Failed to create facility for facilityId ${resourceId}:`, error); + throw error; // Rethrow the error to propagate it + } }; @@ -1059,7 +1078,7 @@ async function createRelatedEntity( mappingArray.push(mappingObject) } } - const mappingObject: any = { mappingArray: mappingArray, CampaignDetails: CampaignDetails, RequestInfo: requestBody?.RequestInfo , parentCampaign: requestBody?.parentCampaign } + const mappingObject: any = { mappingArray: mappingArray, CampaignDetails: CampaignDetails, RequestInfo: requestBody?.RequestInfo, parentCampaign: requestBody?.parentCampaign } await processMapping(mappingObject) } @@ -1466,7 +1485,7 @@ export { getMDMSV1Data, callMdmsTypeSchema, getSheetDataFromWorksheet, - createStaffHelper, + createProjectStaffHelper, createProjectFacilityHelper, createProjectResourceHelper, createAndUploadJsonFile, getConfigurableColumnHeadersBasedOnCampaignTypeForBoundaryManagement diff --git a/health-services/project-factory/src/server/config/index.ts b/health-services/project-factory/src/server/config/index.ts index 1b5234f9afb..d34361809e9 100644 --- a/health-services/project-factory/src/server/config/index.ts +++ b/health-services/project-factory/src/server/config/index.ts @@ -17,6 +17,7 @@ const getDBSchemaName = (dbSchema = "") => { } // Configuration object containing various environment variables const config = { + batchSize:100, cacheTime: 300, isProduction: process.env ? true : false, token: "", // add default token if core services are not port forwarded diff --git a/health-services/project-factory/src/server/service/dataManageService.ts b/health-services/project-factory/src/server/service/dataManageService.ts index 2e87d81a612..db8d4d1924d 100644 --- a/health-services/project-factory/src/server/service/dataManageService.ts +++ b/health-services/project-factory/src/server/service/dataManageService.ts @@ -47,7 +47,7 @@ const downloadDataService = async (request: express.Request) => { tenantId: request?.query?.tenantId, forceUpdate: 'true', hierarchyType: request?.query?.hierarchyType, - campaignId :request?.query?.campaignId, + campaignId: request?.query?.campaignId, }; const newRequestToGenerate = replicateRequest(request, newRequestBody, params); // Added auto generate since no previous generate request found @@ -60,11 +60,11 @@ const downloadDataService = async (request: express.Request) => { // Send response with resource details if (resourceDetails != null && responseData != null && responseData.length > 0) { responseData[0].additionalDetails = { - ...responseData[0].additionalDetails, + ...responseData[0].additionalDetails, ...resourceDetails.additionalDetails // Spread the properties of resourceDetails.additionalDetails }; } - + return responseData; } @@ -94,14 +94,14 @@ const getBoundaryDataService = async ( logger.info("NO CACHE FOUND :: REQUEST :: " + cacheKey); } const workbook = getNewExcelWorkbook(); - const localizationMapHierarchy = hierarchyType && await getLocalizedMessagesHandler(request, request?.query?.tenantId, getLocalisationModuleName(hierarchyType),true); + const localizationMapHierarchy = hierarchyType && await getLocalizedMessagesHandler(request, request?.query?.tenantId, getLocalisationModuleName(hierarchyType), true); const localizationMapModule = await getLocalizedMessagesHandler(request, request?.query?.tenantId); const localizationMap = { ...localizationMapHierarchy, ...localizationMapModule }; // Retrieve boundary sheet data const boundarySheetData: any = await getBoundarySheetData(request, localizationMap); const localizedBoundaryTab = getLocalizedName(getBoundaryTabName(), localizationMap); const boundarySheet = workbook.addWorksheet(localizedBoundaryTab); - addDataToSheet(request,boundarySheet, boundarySheetData, '93C47D', 40, true); + addDataToSheet(request, boundarySheet, boundarySheetData, '93C47D', 40, true); const boundaryFileDetails: any = await createAndUploadFile(workbook, request); // Return boundary file details logger.info("RETURNS THE BOUNDARY RESPONSE"); @@ -120,7 +120,10 @@ const getBoundaryDataService = async ( const createDataService = async (request: any) => { - const localizationMap = await getLocalizedMessagesHandler(request, request?.body?.ResourceDetails?.tenantId); + const hierarchyType = request?.body?.ResourceDetails?.hierarchyType; + const localizationMapHierarchy = hierarchyType && await getLocalizedMessagesHandler(request, request?.body?.ResourceDetails?.tenantId, getLocalisationModuleName(hierarchyType), true); + const localizationMapModule = await getLocalizedMessagesHandler(request, request?.body?.ResourceDetails?.tenantId); + const localizationMap = { ...localizationMapHierarchy, ...localizationMapModule }; // Validate the create request logger.info("Validating data create request") await validateCreateRequest(request, localizationMap); diff --git a/health-services/project-factory/src/server/utils/campaignMappingUtils.ts b/health-services/project-factory/src/server/utils/campaignMappingUtils.ts index fcd712a6a5e..f6226918f76 100644 --- a/health-services/project-factory/src/server/utils/campaignMappingUtils.ts +++ b/health-services/project-factory/src/server/utils/campaignMappingUtils.ts @@ -9,7 +9,7 @@ import { campaignStatuses, resourceDataStatuses } from "../config/constants"; import { createCampaignService, searchProjectTypeCampaignService } from "../service/campaignManageService"; import { persistTrack } from "./processTrackUtils"; import { processTrackTypes, processTrackStatuses } from "../config/constants"; -import { createProjectFacilityHelper, createProjectResourceHelper, createStaffHelper } from "../api/genericApis"; +import { createProjectFacilityHelper, createProjectResourceHelper, createProjectStaffHelper } from "../api/genericApis"; import { buildSearchCriteria, delinkAndLinkResourcesWithProjectCorrespondingToGivenBoundary, processResources } from "./onGoingCampaignUpdateUtils"; import { searchDataService } from "../service/dataManageService"; import { getHierarchy } from "../api/campaignApis"; @@ -562,18 +562,17 @@ export async function handleCampaignMapping(messageObject: any) { } } -export async function handleStaffMapping(mappingArray: any[], campaignId: string, messageObject: any) { +export async function handleStaffMapping(mappingArray: any[], campaignId: string, messageObject: any, type: string) { await persistTrack(campaignId, processTrackTypes.staffMapping, processTrackStatuses.inprogress); try { - const promises = [] - logger.debug("Array of staff mapping: " + getFormattedStringForDebug(mappingArray)); - for (const staffMapping of mappingArray) { - const { resource, projectId, resouceBody, tenantId, startDate, endDate } = staffMapping; - for (const resourceId of resource?.resourceIds) { - promises.push(createStaffHelper(resourceId, projectId, resouceBody, tenantId, startDate, endDate)) - } - } - await Promise.all(promises); + logger.debug(`staff mapping count: ${mappingArray.length}`); + await processResourceOrFacilityOrUserMappingsInBatches(type, mappingArray, config?.batchSize || 100); + // for (const staffMapping of mappingArray) { + // const { resource, projectId, resouceBody, tenantId, startDate, endDate } = staffMapping; + // for (const resourceId of resource?.resourceIds) { + // promises.push(createStaffHelper(resourceId, projectId, resouceBody, tenantId, startDate, endDate)) + // } + // } } catch (error: any) { logger.error("Error in staff mapping: " + error); await persistTrack(campaignId, processTrackTypes.staffMapping, processTrackStatuses.failed, { error: String((error?.message + (error?.description ? ` : ${error?.description}` : '')) || error) }); @@ -583,18 +582,63 @@ export async function handleStaffMapping(mappingArray: any[], campaignId: string await persistTrack(campaignId, processTrackTypes.staffMapping, processTrackStatuses.completed); } -export async function handleResourceMapping(mappingArray: any, campaignId: any, messageObject: any) { - await persistTrack(campaignId, processTrackTypes.resourceMapping, processTrackStatuses.inprogress); - try { - const promises = [] - logger.debug("Arrray of resource mapping: " + getFormattedStringForDebug(mappingArray)); - for (const mapping of mappingArray) { - const { resource, projectId, resouceBody, tenantId, startDate, endDate } = mapping; - for (const resourceId of resource?.resourceIds) { - promises.push(createProjectResourceHelper(resourceId, projectId, resouceBody, tenantId, startDate, endDate)); +async function processResourceOrFacilityOrUserMappingsInBatches(type: string, mappingArray: any, batchSize: number) { + logger.info("Processing resource mappings in batches..."); + let promises: Promise[] = []; + let totalCreated = 0; // To keep track of the total number of created resources + let batchCount = 0; // To log batch-wise progress + // Determine the helper function to use based on the type + let createHelperFn: any; + if (type === 'resource') { + createHelperFn = createProjectResourceHelper; + } else if (type === 'staff') { + createHelperFn = createProjectStaffHelper; + } else if (type === 'facility') { + createHelperFn = createProjectFacilityHelper; + } else { + logger.error(`Unsupported type: ${type}`); + return; // Exit the function if the type is unsupported + } + + for (const mapping of mappingArray) { + const { resource, projectId, resouceBody, tenantId, startDate, endDate } = mapping; + + for (const resourceId of resource?.resourceIds || []) { + promises.push( + createHelperFn(resourceId, projectId, resouceBody, tenantId, startDate, endDate).then(() => { + totalCreated++; + }) + ); + + if (promises.length >= batchSize) { + batchCount++; + logger.info(`Processing batch ${batchCount} with ${promises.length} promises.`); + try { + await Promise.all(promises); // Wait for all promises in the current batch + } catch (error) { + logger.error(`Batch ${batchCount} failed:`, error); + throw error; // Ensure any error in the batch is propagated + } promises = []; // Reset the array for the next batch } } + } + + // Process any remaining promises + if (promises.length > 0) { + batchCount++; + logger.info(`Processing final batch ${batchCount} with ${promises.length} promises.`); await Promise.all(promises); + } + + logger.info(`Processing completed. Total resources created: ${totalCreated}`); +} + + +export async function handleResourceMapping(mappingArray: any[], campaignId: any, messageObject: any, type: string) { + await persistTrack(campaignId, processTrackTypes.resourceMapping, processTrackStatuses.inprogress); + try { + logger.debug(`Resource mapping count: ${mappingArray.length}`); + await processResourceOrFacilityOrUserMappingsInBatches(type, mappingArray, config?.batchSize || 100); } catch (error: any) { logger.error("Error in resource mapping: " + error); await persistTrack(campaignId, processTrackTypes.resourceMapping, processTrackStatuses.failed, { error: String((error?.message + (error?.description ? ` : ${error?.description}` : '')) || error) }); @@ -604,18 +648,17 @@ export async function handleResourceMapping(mappingArray: any, campaignId: any, await persistTrack(campaignId, processTrackTypes.resourceMapping, processTrackStatuses.completed); } -export async function handleFacilityMapping(mappingArray: any, campaignId: any, messageObject: any) { +export async function handleFacilityMapping(mappingArray: any, campaignId: any, messageObject: any, type: string) { await persistTrack(campaignId, processTrackTypes.facilityMapping, processTrackStatuses.inprogress); try { - const promises = [] - logger.debug("Array of facility mapping: " + getFormattedStringForDebug(mappingArray)); - for (const mapping of mappingArray) { - const { resource, projectId, resouceBody, tenantId } = mapping; - for (const resourceId of resource?.resourceIds) { - promises.push(createProjectFacilityHelper(resourceId, projectId, resouceBody, tenantId)); - } - } - await Promise.all(promises); + logger.debug(`facility mapping count: ${mappingArray.length}`); + // for (const mapping of mappingArray) { + // const { resource, projectId, resouceBody, tenantId } = mapping; + // for (const resourceId of resource?.resourceIds) { + // promises.push(createProjectFacilityHelper(resourceId, projectId, resouceBody, tenantId)); + // } + // } + await processResourceOrFacilityOrUserMappingsInBatches(type, mappingArray, config?.batchSize || 100); } catch (error: any) { logger.error("Error in facility mapping: " + error); await persistTrack(campaignId, processTrackTypes.facilityMapping, processTrackStatuses.failed, { error: String((error?.message + (error?.description ? ` : ${error?.description}` : '')) || error) }); @@ -631,9 +674,9 @@ export async function processMapping(mappingObject: any) { const resourceMappingArray = mappingObject?.mappingArray?.filter((mappingObject: any) => mappingObject?.type == "resource"); const facilityMappingArray = mappingObject?.mappingArray?.filter((mappingObject: any) => mappingObject?.type == "facility"); const staffMappingArray = mappingObject?.mappingArray?.filter((mappingObject: any) => mappingObject?.type == "staff"); - await handleResourceMapping(resourceMappingArray, mappingObject?.CampaignDetails?.id, mappingObject); - await handleFacilityMapping(facilityMappingArray, mappingObject?.CampaignDetails?.id, mappingObject); - await handleStaffMapping(staffMappingArray, mappingObject?.CampaignDetails?.id, mappingObject); + await handleResourceMapping(resourceMappingArray, mappingObject?.CampaignDetails?.id, mappingObject, "resource"); + await handleFacilityMapping(facilityMappingArray, mappingObject?.CampaignDetails?.id, mappingObject, "facility"); + await handleStaffMapping(staffMappingArray, mappingObject?.CampaignDetails?.id, mappingObject, "staff"); } logger.info("Mapping completed successfully for campaign: " + mappingObject?.CampaignDetails?.id); mappingObject.CampaignDetails.status = campaignStatuses.inprogress diff --git a/health-services/project-factory/src/server/validators/campaignValidators.ts b/health-services/project-factory/src/server/validators/campaignValidators.ts index 40dc426b5dd..4fa9c08e1cb 100644 --- a/health-services/project-factory/src/server/validators/campaignValidators.ts +++ b/health-services/project-factory/src/server/validators/campaignValidators.ts @@ -1369,12 +1369,12 @@ function validateAllDistrictTabsPresentOrNot(request: any, dataFromSheet: any, d const MissingDistricts: any = []; const campaignBoundaries = request?.body?.campaignBoundaries; if (campaignBoundaries && campaignBoundaries?.length > 0) { - const districtsUnlocalised = campaignBoundaries + const districtsLocalised = campaignBoundaries .filter((data: any) => getLocalizedName(`${request?.body?.ResourceDetails?.hierarchyType}_${data.type.toUpperCase()}`, localizationMap).toLocaleLowerCase() == differentTabsBasedOnLevel.toLowerCase()) .map((data: any) => getLocalizedName(data?.code, localizationMap)) || []; tabsOfDistrict.forEach((tab: any) => { - if (!districtsUnlocalised.includes(tab)) { + if (!districtsLocalised.includes(tab)) { MissingDistricts.push(tab); } });