From 218118a66b81f0c9e73e03a898db19b9e9258218 Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Thu, 25 Jul 2024 08:48:10 -0400 Subject: [PATCH 01/10] Fixed JM not reflecting changes when changes are made, added clear option to clear filters even when filters are hidden --- .../jobMonitoring/JobMonitoringFilters.jsx | 23 ++++++++++++++++--- .../application/jobMonitoring/index.jsx | 9 ++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx index 09682fcb..4ba9657b 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx @@ -35,7 +35,7 @@ function NotificationTableFilters({ const [domainOptions, setDomainOptions] = useState([]); const [productOptions, setProductOptions] = useState([]); const [frequencyOptions, setFrequencyOptions] = useState([]); - const [filterCount, setFilterCount] = useState(4); + const [filterCount, setFilterCount] = useState(0); //Effects useEffect(() => { @@ -151,6 +151,17 @@ function NotificationTableFilters({ setFiltersVisible(true); }; + // Clear filters when clear is clicked + const clearFilters = () => { + form.resetFields(); + setFilterCount(0); + setFilters({}); + // If exists remove jMFilters from local storage + if (localStorage.getItem('jMFilters')) { + localStorage.removeItem('jMFilters'); + } + }; + //JSX return (
@@ -234,9 +245,15 @@ function NotificationTableFilters({ {filterCount > 0 && !filtersVisible && (
-
+
{`${filterCount} filter(s) active`} - - View + + - View + + + {' '} + | Clear +
)} diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx index 7986503b..f9eb8df6 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx @@ -138,7 +138,7 @@ function JobMonitoring() { } }, [editingData, duplicatingData]); - // Get all teams hook, monitoring type ID, Filters from local storage + // Get monitoring type ID, Filters from local storage useEffect(() => { // Get monitoringType id for job monitoring (async () => { @@ -199,7 +199,7 @@ function JobMonitoring() { // When filterChange filter the job monitorings useEffect(() => { if (jobMonitorings.length === 0) return; - if (Object.keys(filters).length < 1) return; + // if (Object.keys(filters).length < 1) return; const { approvalStatus, activeStatus, domain, frequency, product } = filters; // Convert activeStatus to boolean @@ -210,7 +210,7 @@ function JobMonitoring() { activeStatusBool = false; } - const filteredJobMonitorings = jobMonitorings.filter((jobMonitoring) => { + const filteredJm = jobMonitorings.filter((jobMonitoring) => { let include = true; const currentDomain = jobMonitoring?.metaData?.asrSpecificMetaData?.domain; const currentProduct = jobMonitoring?.metaData?.asrSpecificMetaData?.productCategory; @@ -237,7 +237,7 @@ function JobMonitoring() { return include; }); - setFilteredJobMonitoring(filteredJobMonitorings); + setFilteredJobMonitoring(filteredJm); }, [filters, jobMonitorings]); // Function reset states when modal is closed @@ -385,6 +385,7 @@ function JobMonitoring() { allInputs = { ...allInputs, metaData }; const responseData = await createJobMonitoring({ inputData: allInputs }); + setJobMonitorings([responseData, ...jobMonitorings]); message.success('Job monitoring saved successfully'); From 9cc37fdf244bbced72241cd5bf2e6d6a58defbd5 Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Thu, 25 Jul 2024 09:31:48 -0400 Subject: [PATCH 02/10] JM name conflict with soft delted monitoring name breaking page issue fixed --- .../application/jobMonitoring/jobMonitoringUtils.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/jobMonitoringUtils.js b/Tombolo/client-reactjs/src/components/application/jobMonitoring/jobMonitoringUtils.js index 93e419a0..7eeae558 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/jobMonitoringUtils.js +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/jobMonitoringUtils.js @@ -12,7 +12,7 @@ export const createJobMonitoring = async ({ inputData }) => { const response = await fetch(`/api/jobmonitoring`, payload); if (!response.ok) { - return message.error('Failed to save job monitoring'); + throw new Error('Failed to save job monitoring'); } const data = await response.json(); @@ -76,7 +76,7 @@ export const updateSelectedMonitoring = async ({ updatedData }) => { const response = await fetch(`/api/jobmonitoring/`, payload); if (!response.ok) { - return message.error('Failed to update job monitoring'); + throw new Error('Failed to update job monitoring'); } const data = await response.json(); @@ -94,7 +94,7 @@ export const handleDeleteJobMonitoring = async ({ id, jobMonitorings, setJobMoni const response = await fetch(`/api/jobmonitoring/${id}`, payload); if (!response.ok) { - return message.error('Failed to delete job monitoring'); + throw new Error('Failed to delete job monitoring'); } // Set job monitorings @@ -102,7 +102,7 @@ export const handleDeleteJobMonitoring = async ({ id, jobMonitorings, setJobMoni setJobMonitorings(filteredJobMonitorings); message.success('Job monitoring deleted successfully'); } catch (err) { - message.error(err.message); + throw new Error(err.message); } }; From 0bbcd43d96cded2354fe08e68207c7887d891321 Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Thu, 25 Jul 2024 10:52:30 -0400 Subject: [PATCH 03/10] Inline doc for jobName pattern fixed --- .../src/components/userGuides/JobNamePattern.jsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx b/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx index d0173e2b..a0f8770b 100644 --- a/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx +++ b/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx @@ -106,8 +106,9 @@ function JobNamePattern() {

Invalid Job Name

- If an invalid job name is provided, the system will handle it in the same way as an invalid host name. Please - see the Host field notes for details. + If an invalid job name or pattern is provided, Tombolo won't be able to monitor the job correctly. It may + search for a non-existent name or mistakenly monitor a wrong job that matches the pattern. Therefore, please + ensure the name or pattern you provide is correct.

); From 9c22a614531be8104c851f95f9146b9cabd4f5ee Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Thu, 1 Aug 2024 11:15:56 -0400 Subject: [PATCH 04/10] Added loading component while the jobs are loading, fixed other issues related to job monitoring such as date not properly formatting etc --- .../jobMonitoring/JobMonitoringTable.jsx | 2 + .../application/jobMonitoring/index.jsx | 5 +- .../monitorIntermediateStateJobs.js | 95 +++++++++---------- .../jobMonitoring/monitorJobPunctuality.js | 32 ++++--- .../server/jobs/jobMonitoring/monitorJobs.js | 65 +++++++------ .../jobs/jobMonitoring/monitorJobsUtil.js | 5 + 6 files changed, 108 insertions(+), 96 deletions(-) diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx index ec92cb00..2e9c7455 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx @@ -39,6 +39,7 @@ const JobMonitoringTable = ({ setSelectedRows, domains, allProductCategories, + filteringJobs, }) => { //Redux const { @@ -279,6 +280,7 @@ const JobMonitoringTable = ({ return ( { + setFilteringJobs(true); if (jobMonitorings.length === 0) return; // if (Object.keys(filters).length < 1) return; const { approvalStatus, activeStatus, domain, frequency, product } = filters; @@ -238,6 +239,7 @@ function JobMonitoring() { }); setFilteredJobMonitoring(filteredJm); + setFilteringJobs(false); }, [filters, jobMonitorings]); // Function reset states when modal is closed @@ -649,6 +651,7 @@ function JobMonitoring() { domains={domains} productCategories={productCategories} allProductCategories={allProductCategories} + filteringJobs={filteringJobs} /> = severityThreshHold && severeEmailRecipients; + // WU now in state such as failed, aborted etc if (notificationConditionLowerCase.includes(State)) { @@ -252,10 +259,9 @@ const Integrations = models.integrations; "Job Name/Filter": jobNamePattern, "Returned Job": wu.Jobname, State: wu.State, - "Discovered at": - new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString() || now.toISOString(), + "Discovered at": findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }, notificationId: generateNotificationId({ notificationPrefix, @@ -267,27 +273,21 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), - lastLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), + firstLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), + lastLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }); - // notificationPayload.wuId = wu.Wuid; + notificationPayload.wuId = wu.Wuid; notificationsToBeQueued.push(notificationPayload); // NOC email notification if severity is high - if ( - severity >= severityThreshHold && - severeEmailRecipients - ) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; - notificationPayload.metaData.mainRecipients = - severeEmailRecipients; + if (sendAlertToNoc) { + notificationPayload.metaData.notificationDescription = nocAlertDescription; + notificationPayload.metaData.mainRecipients = severeEmailRecipients; delete notificationPayload.metaData.cc; notificationsToBeQueued.push(notificationPayload); } @@ -300,6 +300,7 @@ const Integrations = models.integrations; currentTimeToWindowRelation === "after" && requireComplete === true ) { + // Add new State to the WU wu.State = _.capitalize(State); @@ -325,10 +326,9 @@ const Integrations = models.integrations; "Job Name/Filter": jobNamePattern, "Returned Job": wu.Jobname, State: wu.State, - "Discovered at": - new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString() || now.toISOString(), + "Discovered at": findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }, notificationId: generateNotificationId({ notificationPrefix, @@ -340,27 +340,23 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), - lastLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), + firstLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), + lastLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }); notificationsToBeQueued.push(notificationPayload); // NOC email notification if severity is high - if (severity >= severityThreshHold && severeEmailRecipients) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; - notificationPayload.metaData.mainRecipients = - severeEmailRecipients; + if (sendAlertToNoc) { + notificationPayload.metaData.notificationDescription = nocAlertDescription; + notificationPayload.metaData.mainRecipients = severeEmailRecipients; delete notificationPayload.metaData.cc; notificationsToBeQueued.push(notificationPayload); } - wuNoLongerInIntermediateState.push(wu.Wuid); } // IF the job is still in intermediate state and the current time is within the run window @@ -368,6 +364,7 @@ const Integrations = models.integrations; intermediateStates.includes(State) && currentTimeToWindowRelation === "within" ) { + // If the State has changed from last time it was checked, update monitoring needs to be updated with new state if (wu.State !== State) { wuWithNewIntermediateState[wu.Wuid] = State; @@ -378,7 +375,7 @@ const Integrations = models.integrations; } } catch (err) { logger.error( - `WUId - ${wu.Wuid} - Cluster ${cluster_id}: ${err.message}` + `Monitor Intermediate Jobs. WUId - ${wu.Wuid} - Cluster ${cluster_id}: ${err.message}` ); } } @@ -429,7 +426,7 @@ const Integrations = models.integrations; { where: { id } } ); } catch (error) { - logger.error(`Error updating log with id ${log.id}:`, error); + logger.error(`Intermediate State Jobs - Error updating log with id ${log.id}:`, error); } } } catch (err) { diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js index 2583ecd4..4ebd74e7 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js @@ -38,25 +38,27 @@ const Integrations = models.integrations; // Find severity level (For ASR ) - based on that determine when to send out notifications let severityThreshHold = 0; - let severeEmailRecipients; + let severeEmailRecipients = null; try{ const {id : integrationId} = await Integrations.findOne({where: {name: "ASR"}, raw: true}); - if(integrationId){ - // Get integration mapping with integration details - const integrationMapping = await IntegrationMapping.findOne({ - where : {integration_id : integrationId}, raw: true - }); - - const { - metaData: { - nocAlerts: { severityLevelForNocAlerts, emailContacts }, - }, - } = integrationMapping; - severityThreshHold = severityLevelForNocAlerts; - severeEmailRecipients = emailContacts; - } + if(integrationId){ + // Get integration mapping with integration details + const integrationMapping = await IntegrationMapping.findOne({ + where : {integration_id : integrationId}, raw: true + }); + + if(integrationMapping){ + const { + metaData: { + nocAlerts: { severityLevelForNocAlerts, emailContacts }, + }, + } = integrationMapping; + severityThreshHold = severityLevelForNocAlerts; + severeEmailRecipients = emailContacts; + } + } }catch(error){ logger.error(`Job Punctuality Monitoring : Error while getting integration level severity threshold: ${error.message}`); } diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js index 0a27a372..e45ba1d1 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js @@ -14,6 +14,8 @@ const { generateNotificationId, getProductCategory, getDomain, + findLocalDateTimeAtCluster, + nocAlertDescription, } = require("./monitorJobsUtil"); const e = require("express"); @@ -53,7 +55,7 @@ const Integrations = models.integrations; // Find severity level (For ASR ) - based on that determine when to send out notifications let severityThreshHold = 0; - let severeEmailRecipients; + let severeEmailRecipients = null; try { const { id: integrationId } = await Integrations.findOne({ @@ -68,13 +70,16 @@ const Integrations = models.integrations; raw: true, }); - const { - metaData: { - nocAlerts: { severityLevelForNocAlerts, emailContacts }, - }, - } = integrationMapping; - severityThreshHold = severityLevelForNocAlerts; - severeEmailRecipients = emailContacts; + + if(integrationMapping){ + const { + metaData: { + nocAlerts: { severityLevelForNocAlerts, emailContacts }, + }, + } = integrationMapping; + severityThreshHold = severityLevelForNocAlerts; + severeEmailRecipients = emailContacts; + } } } catch (error) { logger.error( @@ -154,6 +159,7 @@ const Integrations = models.integrations; /* Fetch basic information for all work units per cluster */ const wuBasicInfoByCluster = {}; + const failedToReachClusters = []; for (let clusterInfo of clustersInfo) { try { const wuService = new WorkunitsService({ @@ -174,9 +180,8 @@ const Integrations = models.integrations; wuBasicInfoByCluster[clusterInfo.id] = [...wuWithClusterIds]; } catch (err) { - logger.error( - `Job monitoring - Error while reaching out to cluster ${clusterInfo.id} : ${err}` - ); + failedToReachClusters.push(clusterInfo.id); + logger.error(`Job monitoring - Error while reaching out to cluster ${clusterInfo.id} : ${err}`); } } @@ -371,10 +376,9 @@ const Integrations = models.integrations; Cluster: clusterInfoObj[clusterId].name || "", "Job Name/Filter": jobName, "Returned Job": wu.Jobname, - "Discovered at": - new Date( - now + clusterInfoObj[clusterId].timezone_offset * 60 * 1000 - ).toISOString() || now.toISOString(), + "Discovered at": findLocalDateTimeAtCluster( + clusterInfoObj[clusterId].timezone_offset + ), State: wu.State, }, notificationId: generateNotificationId({ @@ -387,12 +391,12 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date( - now + clusterInfoObj[clusterId].timezone_offset * 60 * 1000 - ).toISOString(), - lastLogged: new Date( - now + clusterInfoObj[clusterId].timezone_offset * 60 * 1000 - ).toISOString(), + firstLogged: findLocalDateTimeAtCluster( + clusterInfoObj[clusterId].timezone_offset + ), + lastLogged: findLocalDateTimeAtCluster( + clusterInfoObj[clusterId].timezone_offset + ), }); //Create notification queue @@ -400,17 +404,20 @@ const Integrations = models.integrations; // If severity is above threshold, send out NOC notification if (severity >= severityThreshHold && severeEmailRecipients) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; + notificationPayload.metaData.notificationDescription = nocAlertDescription; notificationPayload.metaData.mainRecipients = severeEmailRecipients; delete notificationPayload.metaData.cc; await NotificationQueue.create(notificationPayload); } } + // If failed to reach cluster, do not update last monitored time in monitoring logs + const scannedClusters = clusterIds.filter( + (id) => !failedToReachClusters.includes(id) + ); + // Update monitoring logs - for (let id of clusterIds) { + for (let id of scannedClusters) { try { //Get existing metadata const log = await MonitoringLogs.findOne({ @@ -456,6 +463,7 @@ const Integrations = models.integrations; ); } } + } catch (err) { logger.error(err); } finally { @@ -463,9 +471,4 @@ const Integrations = models.integrations; if (parentPort) parentPort.postMessage("done"); else process.exit(0); } -})(); - - - -// TODO - Start from Intermediate -// TODO - All monitorings - update last run details +})(); \ No newline at end of file diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js b/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js index 68df5331..51595717 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js @@ -657,6 +657,10 @@ function generateJobName({ pattern, timezone_offset = 0 }) { // console.log( generateJobName({pattern: "* Test",timezone_offset: 0})); // console.log(generateJobName({ pattern: "Launch "})); +const nocAlertDescription = `[SEV TICKET REQUEST] + The following issue has been identified via automation. + Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; + module.exports = { matchJobName, findStartAndEndTimes, @@ -671,4 +675,5 @@ module.exports = { intermediateStates, getProductCategory, getDomain, + nocAlertDescription, }; From 04ecd66b25a4a9fb828825ca60a6be148bbb819d Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Mon, 5 Aug 2024 08:03:47 -0400 Subject: [PATCH 05/10] temp commit to switch to another branch --- .../jobMonitoring/MonitoringDetailsModal.jsx | 19 ++++- .../jobMonitoring/SchedulePicker.jsx | 2 +- .../application/jobMonitoring/index.jsx | 4 +- .../jobSchedularMethods/jobMonitoring.js | 12 +-- .../monitorIntermediateStateJobs.js | 62 ++++++++++---- .../jobMonitoring/monitorJobPunctuality.js | 80 +++++++++++++++---- .../server/jobs/jobMonitoring/monitorJobs.js | 29 +++++-- .../jobs/jobMonitoring/monitorJobsUtil.js | 63 ++++++++++----- .../processEmailNotifications.js | 6 +- 9 files changed, 206 insertions(+), 71 deletions(-) diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/MonitoringDetailsModal.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/MonitoringDetailsModal.jsx index 426227e7..011eb43b 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/MonitoringDetailsModal.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/MonitoringDetailsModal.jsx @@ -104,7 +104,7 @@ function MonitoringDetailsModal({ )} )} - {asrSpecificMetaData?.severity && ( + {asrSpecificMetaData?.severity !== undefined && asrSpecificMetaData?.severity !== null && ( {asrSpecificMetaData.severity} )} {/* ---------NOTIFICATION TRIGGERS AND CONTACTS --------------------------------------------- */} @@ -213,22 +213,33 @@ function MonitoringDetailsModal({ export default MonitoringDetailsModal; +// Interpret run window +const interpretRunWindow = (schedule) => { + const runWindow = schedule[0].runWindow || ''; + if (runWindow === '') { + return ''; + } else if (runWindow === 'daily') return 'Anytime'; + else { + return _.capitalize(runWindow); + } +}; + //Generate tags for schedule const generateTagsForSchedule = (schedule) => { const tags = []; schedule.forEach((s) => { if (s.frequency === 'daily') { - tags.push('Everyday'); + tags.push(interpretRunWindow(schedule)); } if (s.frequency === 'weekly') { - let tempData = 'Every week on'; + let tempData = `Every Week ${interpretRunWindow(schedule)} on`; s.days.forEach((d, i) => { tempData += ` ${getDayLabel(d)} ${i < s.days.length - 1 ? ',' : ''}`; }); tags.push(tempData); } if (s.scheduleBy === 'dates') { - let tempData = 'Every month on'; + let tempData = `Every month ${interpretRunWindow(schedule)}`; s.dates.forEach((d, i) => { tempData += ` ${getDateLabel(d)} ${i < s.dates.length - 1 ? ',' : ''}`; }); diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx index 7bb93284..01eaaca3 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx @@ -10,7 +10,7 @@ const dailyRunWindowAndIntervals = [ { label: 'Morning (00:00 - 11:59)', value: 'morning' }, { label: 'Afternoon (12:00 - 23:59)', value: 'afternoon' }, { label: 'Overnight (Prev 12:00 - Current day 12:00)', value: 'overnight' }, - { label: 'Every 2 Days', value: 'every2Days' }, + // { label: 'Every 2 Days', value: 'every2Days' }, ]; // Daily schedule options diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx index e635617d..3c215466 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx @@ -199,7 +199,9 @@ function JobMonitoring() { // When filterChange filter the job monitorings useEffect(() => { setFilteringJobs(true); - if (jobMonitorings.length === 0) return; + if (jobMonitorings.length === 0) { + setFilteringJobs(false); + } // if (Object.keys(filters).length < 1) return; const { approvalStatus, activeStatus, domain, frequency, product } = filters; diff --git a/Tombolo/server/jobSchedularMethods/jobMonitoring.js b/Tombolo/server/jobSchedularMethods/jobMonitoring.js index 20f80fce..fb354615 100644 --- a/Tombolo/server/jobSchedularMethods/jobMonitoring.js +++ b/Tombolo/server/jobSchedularMethods/jobMonitoring.js @@ -26,8 +26,8 @@ async function startJobMonitoring() { let jobName = "job-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - //interval: "10s", // For development - interval: humanReadableIntervalForJobMonitoring, + interval: "10s", // For development + // interval: humanReadableIntervalForJobMonitoring, path: path.join( __dirname, "..", @@ -65,8 +65,8 @@ async function startIntermediateJobsMonitoring() { let jobName = "intermediate-state-jobs-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - //interval: "20s", // For development - interval: humanReadableIntervalForIntermediateJobMonitoring, + interval: "20s", // For development + // interval: humanReadableIntervalForIntermediateJobMonitoring, path: path.join( __dirname, "..", @@ -103,8 +103,8 @@ async function startJobPunctualityMonitoring() { let jobName = "job-punctuality-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - //interval: "10s", // For development - interval: humanReadableIntervalForJobPunctualityMonitoring, + interval: "10s", // For development + // interval: humanReadableIntervalForJobPunctualityMonitoring, path: path.join( __dirname, "..", diff --git a/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js index 04e4d261..8309a447 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js @@ -216,9 +216,7 @@ const Integrations = models.integrations; try { const info = await wuService.WUInfo({ Wuid: wu.Wuid }); - const { - Workunit: { State }, - } = info; + const {Workunit: { State }} = info; // Check if current time is before, after, within the window const currentTimeToWindowRelation = @@ -234,6 +232,9 @@ const Integrations = models.integrations; // WU now in state such as failed, aborted etc if (notificationConditionLowerCase.includes(State)) { + console.log('------------------------------------------'); + console.dir("Intermediate job failed") + console.log('------------------------------------------'); // Add new State to the WU wu.State = _.capitalize(State); @@ -286,10 +287,15 @@ const Integrations = models.integrations; // NOC email notification if severity is high if (sendAlertToNoc) { - notificationPayload.metaData.notificationDescription = nocAlertDescription; - notificationPayload.metaData.mainRecipients = severeEmailRecipients; - delete notificationPayload.metaData.cc; - notificationsToBeQueued.push(notificationPayload); + const notificationPayloadForNoc = { ...notificationPayload} + notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayloadForNoc.metaData.notificationId = generateNotificationId({ + notificationPrefix, + timezoneOffset: clusterDetail.timezone_offset || 0, + }), + delete notificationPayloadForNoc.metaData.cc; + notificationsToBeQueued.push(notificationPayloadForNoc); } wuNoLongerInIntermediateState.push(wu.Wuid); @@ -300,6 +306,9 @@ const Integrations = models.integrations; currentTimeToWindowRelation === "after" && requireComplete === true ) { + console.log('------------------------------------------'); + console.dir("Intermediate job passed time") + console.log('------------------------------------------'); // Add new State to the WU wu.State = _.capitalize(State); @@ -347,16 +356,35 @@ const Integrations = models.integrations; clusterDetail.timezone_offset ), }); + + // console.log('-----------Payload 1----------------------'); + // console.dir(notificationPayload) + // console.log('------------------------------------------'); notificationsToBeQueued.push(notificationPayload); // NOC email notification if severity is high if (sendAlertToNoc) { - notificationPayload.metaData.notificationDescription = nocAlertDescription; - notificationPayload.metaData.mainRecipients = severeEmailRecipients; - delete notificationPayload.metaData.cc; - notificationsToBeQueued.push(notificationPayload); + const notificationPayloadForNoc = { ...notificationPayload} + notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayloadForNoc.metaData.notificationId = generateNotificationId({ + notificationPrefix, + timezoneOffset: clusterDetail.timezone_offset || 0, + }), + delete notificationPayloadForNoc.metaData.cc; + + + // console.log("-----------Payload 2----------------------"); + // console.dir(notificationPayload); + // console.log("------------------------------------------"); + + notificationsToBeQueued.push(notificationPayloadForNoc); } + console.log('----------To be queued -------------------'); + console.dir(notificationsToBeQueued) + console.log('------------------------------------------'); + wuNoLongerInIntermediateState.push(wu.Wuid); } // IF the job is still in intermediate state and the current time is within the run window @@ -364,12 +392,18 @@ const Integrations = models.integrations; intermediateStates.includes(State) && currentTimeToWindowRelation === "within" ) { + console.log('------------------------------------------'); + console.dir("STILL IN INTERMEDIATE STATE") + console.log('------------------------------------------'); // If the State has changed from last time it was checked, update monitoring needs to be updated with new state if (wu.State !== State) { wuWithNewIntermediateState[wu.Wuid] = State; } } else { + console.log('------------------------------------------'); + console.dir("COMPLETED") + console.log('------------------------------------------'); // WU in completed state - Remove the WU from the intermediate state wuNoLongerInIntermediateState.push(wu.Wuid); } @@ -386,6 +420,9 @@ const Integrations = models.integrations; // Insert notification in queue for (let notification of notificationsToBeQueued) { + // console.log('-NOTIFICATION LOOP ------------------------'); + // console.dir(notification); + // console.log('------------------------------------------'); await notification_queue.create(notification); } @@ -432,9 +469,6 @@ const Integrations = models.integrations; } catch (err) { logger.error(err); } finally { - logger.debug( - `Job monitoring completed started ${now} and ended at ${new Date()}` - ); if (parentPort) parentPort.postMessage("done"); else process.exit(0); } diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js index 4ebd74e7..f1115c11 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js @@ -1,6 +1,7 @@ const { WorkunitsService } = require("@hpcc-js/comms"); const logger = require("../../config/logger"); const { decryptString } = require("../../utils/cipher"); +const { parentPort } = require("worker_threads"); const { calculateRunOrCompleteByTimes, generateJobName, @@ -8,6 +9,8 @@ const { getProductCategory, getDomain, generateNotificationId, + differenceInMs, + nocAlertDescription, } = require("./monitorJobsUtil"); const models = require("../../models"); @@ -24,7 +27,6 @@ const Integrations = models.integrations; const now = new Date(); // UTC time try { - // Get all job monitorings // Find all active job monitorings. const jobMonitorings = await JobMonitoring.findAll({ where: { isActive: 1, approvalStatus: "Approved" }, @@ -125,11 +127,30 @@ const Integrations = models.integrations; // Job level severity threshold const jobLevelSeverity = asrSpecificMetaData?.severity || 0; + // If job level severity is less than the threshold, check only after the completion time - let backDateInMinutes = 0; - if(jobLevelSeverity < severityThreshHold){ - backDateInMinutes = 1440; // 24 hours + let backDateInMs = 0; + if ( jobLevelSeverity < severityThreshHold) { + let runWindowForJob = null; + if (schedule[0]?.runWindow) { + runWindowForJob = schedule[0].runWindow; + } + + // Calculate the back date in ms + if (runWindowForJob === "overnight") { + backDateInMs = differenceInMs({ + startTime: expectedCompletionTime, + endTime: expectedStartTime, + daysDifference: 1, + }); + } else { + backDateInMs = differenceInMs({ + startTime: expectedCompletionTime, + endTime: expectedStartTime, + daysDifference: 0, + }); + } } // Calculate the run window for the job @@ -138,7 +159,7 @@ const Integrations = models.integrations; timezone_offset: clusterInfo.timezone_offset, expectedStartTime, expectedCompletionTime, - backDateInMinutes, + backDateInMs, }); // If the window null - continue. Job is not expected to run @@ -146,12 +167,13 @@ const Integrations = models.integrations; continue; } - let timePassed = false; + let alertTimePassed = false; let lateByInMinutes = 0; if(jobLevelSeverity < severityThreshHold){ + alertTimePassed = window.end < window.currentTime; lateByInMinutes = Math.floor((window.currentTime - window.end) / 60000); }else{ - timePassed = window.start < window.currentTime; + alertTimePassed = window.start < window.currentTime; lateByInMinutes = Math.floor((window.currentTime - window.start) / 60000); } @@ -161,7 +183,7 @@ const Integrations = models.integrations; } // If the time has not passed, continue - if (!timePassed) { + if (!alertTimePassed) { continue; } @@ -200,6 +222,25 @@ const Integrations = models.integrations; Jobname: translatedJobName, }); + // If a job is overnight, it could potentially have 2 translatedJobName as it can run on 2 different days + if(schedule[0]?.runWindow === 'overnight'){ + const translatedJobNameNextDay = generateJobName({ + pattern: jobNamePattern, + timezone_offset: clusterInfo.timezone_offset, + backDateInDays: 1, + }); + + const { + Workunits: { ECLWorkunit: ECLWorkunitNextDay }, + } = await wuService.WUQuery({ + StartDate: window.start, + EndDate: window.end, + Jobname: translatedJobNameNextDay, + }); + + ECLWorkunit.push(...ECLWorkunitNextDay); + } + // If workunits are found, update the job monitoring and continue if (ECLWorkunit.length > 0) { await JobMonitoring.update( @@ -288,12 +329,16 @@ const Integrations = models.integrations; // NOC email notification if (jobLevelSeverity >= severityThreshHold && severeEmailRecipients) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.` - notificationPayload.metaData.mainRecipients = severeEmailRecipients; - delete notificationPayload.metaData.cc; - await NotificationQueue.create(notificationPayload); + const notificationPayloadForNoc = { ...notificationPayload }; + notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayloadForNoc.metaData.notificationId = + generateNotificationId({ + notificationPrefix, + timezoneOffset: offSet || 0, + }); + delete notificationPayloadForNoc.metaData.cc; + await NotificationQueue.create(notificationPayloadForNoc); } // Update the job monitoring await JobMonitoring.update( @@ -317,11 +362,12 @@ const Integrations = models.integrations; logger.error( `Error while processing jobs for punctuality check ${jobMonitoring.id}: ${error.message}` ); + }finally{ + if (parentPort) parentPort.postMessage("done"); + else process.exit(0); } } } catch (error) { - logger.error( - `Error in job punctuality monitoring script: ${error.message}` - ); + logger.error(`Error in job punctuality monitoring script: ${error.message}`); } })(); \ No newline at end of file diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js index e45ba1d1..495c4f14 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js @@ -195,9 +195,19 @@ const Integrations = models.integrations; } } + console.log('------------------------------------------'); + console.dir("HERE") + console.log(newWorkUnitsFound); + console.log('------------------------------------------'); + // If no new monitoring work units are found, update the monitoring logs and exit if (!newWorkUnitsFound) { - for (let id of clusterIds) { + // If failed to reach cluster, do not update last monitored time in monitoring logs + const scanned_clusters = clusterIds.filter( + (id) => !failedToReachClusters.includes(id) + ); + + for (let id of scanned_clusters) { // grab existing metaData const log = await MonitoringLogs.findOne({ where: { monitoring_type_id: monitoringTypeId, cluster_id: id }, @@ -275,6 +285,9 @@ const Integrations = models.integrations; ); let clusterWUs = wuBasicInfoByCluster[clusterId]; + console.log('---- Cluster wus -------------------------'); + console.dir(clusterWUs) + console.log('------------------------------------------'); const matchedWus = clusterWUs.filter((wu) => { return matchJobName({ @@ -404,10 +417,15 @@ const Integrations = models.integrations; // If severity is above threshold, send out NOC notification if (severity >= severityThreshHold && severeEmailRecipients) { - notificationPayload.metaData.notificationDescription = nocAlertDescription; - notificationPayload.metaData.mainRecipients = severeEmailRecipients; - delete notificationPayload.metaData.cc; - await NotificationQueue.create(notificationPayload); + const notificationPayloadForNoc = { ...notificationPayload }; + notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayload.metaData.notificationId = generateNotificationId({ + notificationPrefix, + timezoneOffset: clusterInfoObj[clusterId].timezone_offset || 0, + }), + delete notificationPayloadForNoc.metaData.cc; + await NotificationQueue.create(notificationPayloadForNoc); } } @@ -467,7 +485,6 @@ const Integrations = models.integrations; } catch (err) { logger.error(err); } finally { - logger.debug(`Job monitoring completed started ${now} and ended at ${new Date()}`); if (parentPort) parentPort.postMessage("done"); else process.exit(0); } diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js b/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js index 51595717..8976fb99 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js @@ -136,6 +136,7 @@ function setTimeToDate(date, time) { //Calculate start and end time given local time at cluster , run window, expected start and completion time function calculateStartAndEndDateTime({localDateTimeAtCluster, runWindow, expectedStartTime, expectedCompletionTime}) { let startAndEnd; + const previousDay = new Date(localDateTimeAtCluster.getTime() - 86400000); if (runWindow === "overnight") { startAndEnd = { start: setTimeToDate(previousDay, expectedStartTime || "12:00"), @@ -167,11 +168,11 @@ function calculateStartAndEndDateTime({localDateTimeAtCluster, runWindow, expect } // Daily jobs -function calculateRunOrCompleteByTimeForDailyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimeForDailyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs = 0}) { const {runWindow} = schedule; const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); let window = {frequency: "daily", currentTime: localDateTimeAtCluster}; @@ -190,14 +191,14 @@ function calculateRunOrCompleteByTimeForWeeklyJobs({ expectedStartTime, expectedCompletionTime, timezone_offset, - backDateInMinutes = 0, + backDateInMs = 0, }) { const { days, runWindow } = schedule; // Find current day at the cluster given timezone offset and this system in utc const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); const adjustedLocalTimeAtCluster = new Date( - localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000 + localDateTimeAtCluster.getTime() - backDateInMs ); const day = adjustedLocalTimeAtCluster.getDay(); const runDay = days.find((d) => d === day.toString()); @@ -219,11 +220,11 @@ function calculateRunOrCompleteByTimeForWeeklyJobs({ } // Monthly jobs -function calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs = 0}) { // Determine if schedule is by date or week and weekday const { scheduleBy } = schedule[0]; const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); let window = { currentTime: localDateTimeAtCluster, @@ -299,12 +300,12 @@ function calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime } // Yearly jobs -function calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs = 0}) { const {scheduleBy} = schedule[0]; // Local date time at cluster const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); // Current month at the cluster const monthAtCluster = adjustedLocalDateTimeAtCluster.getMonth(); @@ -398,13 +399,13 @@ function calculateRunOrCompleteByTimeForCronJobs({ expectedStartTime, expectedCompletionTime, timezone_offset, - backDateInMinutes = 0, + backDateInMs = 0, }) { const cron = schedule[0].cron; // Local date time at cluster const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); // Get the previous and next dates the cron job was supposed to run const interval = cronParser.parseExpression(cron, { @@ -463,21 +464,21 @@ function checkIfCurrentTimeIsWithinRunWindow({start, end, currentTime}) { } // Calculate run and complete by time for a job on cluster's local time -function calculateRunOrCompleteByTimes({schedule, timezone_offset, expectedStartTime, expectedCompletionTime, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimes({schedule, timezone_offset, expectedStartTime, expectedCompletionTime, backDateInMs = 0}) { // determine frequency frequency = schedule[0].frequency; switch (frequency) { case "daily": - return calculateRunOrCompleteByTimeForDailyJobs({schedule: schedule[0],expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForDailyJobs({schedule: schedule[0],expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "weekly": - return calculateRunOrCompleteByTimeForWeeklyJobs({schedule: schedule[0], expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForWeeklyJobs({schedule: schedule[0], expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "monthly": - return calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "yearly": - return calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "cron": - return calculateRunOrCompleteByTimeForCronJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForCronJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); default: throw new Error(`Unknown frequency: ${frequency}`); } @@ -603,7 +604,7 @@ function getDateReplacements(date) { } // Generate job name from job pattern -function generateJobName({ pattern, timezone_offset = 0 }) { +function generateJobName({ pattern, timezone_offset = 0, backDateInDays = 0 }) { let patternCopy = pattern; if (!patternCopy) return ""; @@ -616,7 +617,9 @@ function generateJobName({ pattern, timezone_offset = 0 }) { // No pattern, no adjustments if (dateSubstring === "") { - const date = new Date(Date.now() + timezone_offset * 1000); + let date = new Date(Date.now() + timezone_offset * 1000); + // adjust backdate + date = new Date(date.getTime() - backDateInDays * 86400000); const replacements = getDateReplacements(date); const translatedDate = replacements["%Y"] + replacements["%m"] + replacements["%d"]; @@ -636,9 +639,7 @@ function generateJobName({ pattern, timezone_offset = 0 }) { patternCopy = patternCopy.replace("", ""); - const date = new Date( - Date.now() + timezone_offset * 60000 + adjustment * 86400000 - ); + const date = new Date(Date.now() + timezone_offset * 60000 + adjustment * 86400000); const replacements = getDateReplacements(date); for (const key in replacements) { @@ -661,6 +662,25 @@ const nocAlertDescription = `[SEV TICKET REQUEST] The following issue has been identified via automation. Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; + + +// Given 2 times in HH:MM format, calculate the difference in milliseconds +function differenceInMs({ startTime, endTime, daysDifference }) { + const [hours1, minutes1] = startTime.split(":").map(Number); + const [hours2, minutes2] = endTime.split(":").map(Number); + + const startDate = new Date(); + const endDate = new Date(); + + startDate.setHours(hours1, minutes1, 0, 0); + endDate.setHours(hours2, minutes2, 0, 0); + + startDate.setDate(startDate.getDate() - daysDifference); + + const difference = endDate - startDate; // Convert milliseconds to minutes + return Math.abs(difference); +} + module.exports = { matchJobName, findStartAndEndTimes, @@ -676,4 +696,5 @@ module.exports = { getProductCategory, getDomain, nocAlertDescription, + differenceInMs, }; diff --git a/Tombolo/server/jobs/notifications/processEmailNotifications.js b/Tombolo/server/jobs/notifications/processEmailNotifications.js index 9dd89d2c..bb7e58e6 100644 --- a/Tombolo/server/jobs/notifications/processEmailNotifications.js +++ b/Tombolo/server/jobs/notifications/processEmailNotifications.js @@ -96,7 +96,11 @@ const SentNotification = models.sent_notifications; }; // Send email - await sendEmail({ ...emailPayload }); + const emailResponse = await sendEmail({ ...emailPayload }); + + console.log('------------------------------------------'); + console.dir(emailResponse) + console.log('------------------------------------------'); // Assume success - if no error is thrown successfulDelivery.push(emailPayload); From 68bfba869ccf9a27853b0294005bf9e25a6ce1b6 Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Wed, 14 Aug 2024 08:24:42 -0400 Subject: [PATCH 06/10] removed test logs --- Tombolo/server/jobs/jobMonitoring/monitorJobs.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js index 495c4f14..6c70b41d 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js @@ -285,9 +285,6 @@ const Integrations = models.integrations; ); let clusterWUs = wuBasicInfoByCluster[clusterId]; - console.log('---- Cluster wus -------------------------'); - console.dir(clusterWUs) - console.log('------------------------------------------'); const matchedWus = clusterWUs.filter((wu) => { return matchJobName({ From cd4e1d6579fee7bcc4de68270aba5b9b2f4ecb6e Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Mon, 19 Aug 2024 16:28:01 -0400 Subject: [PATCH 07/10] Removed unnecessary console.logs --- Tombolo/server/jobs/jobMonitoring/monitorJobs.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js index 495c4f14..6c70b41d 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js @@ -285,9 +285,6 @@ const Integrations = models.integrations; ); let clusterWUs = wuBasicInfoByCluster[clusterId]; - console.log('---- Cluster wus -------------------------'); - console.dir(clusterWUs) - console.log('------------------------------------------'); const matchedWus = clusterWUs.filter((wu) => { return matchJobName({ From b70b6dc5cb359a964db8fb657637493c8c0070c6 Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Tue, 20 Aug 2024 15:18:15 -0400 Subject: [PATCH 08/10] temp commit to fix logging issues first --- Tombolo/server/config/logger.js | 10 +-- .../jobMonitoring/monitorJobPunctuality.js | 69 +++++++++++-------- .../server/jobs/jobMonitoring/monitorJobs.js | 5 -- 3 files changed, 47 insertions(+), 37 deletions(-) diff --git a/Tombolo/server/config/logger.js b/Tombolo/server/config/logger.js index e8e81f5f..8ddf8ea1 100644 --- a/Tombolo/server/config/logger.js +++ b/Tombolo/server/config/logger.js @@ -49,10 +49,10 @@ const logger = createLogger({ }); // If we're not in production then also log to the files -// if (process.env.NODE_ENV !== 'production') { -// const settings = {...common, format: getFormat()}; // will write to files same output as to console but no special char for coloring -// logger.add(new transports.File({ ...settings, level: 'http', filename: './logs/combined.log' })); -// logger.add(new transports.File({ ...settings, level: 'error', filename: './logs/error.log' })); // only logger.error() will be written here -// } +if (process.env.NODE_ENV !== 'production') { + const settings = {...common, format: getFormat()}; // will write to files same output as to console but no special char for coloring + logger.add(new transports.File({ ...settings, level: 'http', filename: './logs/combined.log' })); + logger.add(new transports.File({ ...settings, level: 'error', filename: './logs/error.log' })); // only logger.error() will be written here +} module.exports = logger; \ No newline at end of file diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js index f1115c11..304ee730 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js @@ -38,33 +38,6 @@ const Integrations = models.integrations; return; } - // Find severity level (For ASR ) - based on that determine when to send out notifications - let severityThreshHold = 0; - let severeEmailRecipients = null; - - try{ - const {id : integrationId} = await Integrations.findOne({where: {name: "ASR"}, raw: true}); - - if(integrationId){ - // Get integration mapping with integration details - const integrationMapping = await IntegrationMapping.findOne({ - where : {integration_id : integrationId}, raw: true - }); - - if(integrationMapping){ - const { - metaData: { - nocAlerts: { severityLevelForNocAlerts, emailContacts }, - }, - } = integrationMapping; - severityThreshHold = severityLevelForNocAlerts; - severeEmailRecipients = emailContacts; - } - } - }catch(error){ - logger.error(`Job Punctuality Monitoring : Error while getting integration level severity threshold: ${error.message}`); - } - // All clusters const clusters = await Cluster.findAll({ raw: true }); @@ -125,8 +98,46 @@ const Integrations = models.integrations; const { schedule, expectedStartTime, expectedCompletionTime } =metaData; const clusterInfo = clustersObj[clusterId]; + // Find severity level (For ASR ) - based on that determine when to send out notifications + let severityThreshHold = 0; + let severeEmailRecipients = null; + + if(metaData.asrSpecificMetaData){ + try{ + const {id : integrationId} = await Integrations.findOne({where: {name: "ASR"}, raw: true}); + + if(integrationId){ + // Get integration mapping with integration details + const integrationMapping = await IntegrationMapping.findOne({ + where: { + integration_id: integrationId, + application_id: applicationId, + }, + raw: true, + }); + + if(integrationMapping){ + const { + metaData: { + nocAlerts: { severityLevelForNocAlerts, emailContacts }, + }, + } = integrationMapping; + severityThreshHold = severityLevelForNocAlerts; + severeEmailRecipients = emailContacts; + } + } + }catch(error){ + logger.error(`Job Punctuality Monitoring : Error while getting integration level severity threshold: ${error.message}`); + } + } + + // Job level severity threshold const jobLevelSeverity = asrSpecificMetaData?.severity || 0; + + // console.log('---------- SEVERITY ----------------------'); + // console.log(jobLevelSeverity, severityThreshHold) + // console.log('------------------------------------------'); // If job level severity is less than the threshold, check only after the completion time @@ -153,6 +164,8 @@ const Integrations = models.integrations; } } + console.log("-------------------------------------->") + // Calculate the run window for the job const window = calculateRunOrCompleteByTimes({ schedule, @@ -189,6 +202,8 @@ const Integrations = models.integrations; // Check if notification has been sent out for this job, for the current window const jobPunctualityDetails = lastJobRunDetails?.jobPunctualityDetails; + + console.log("------", lastJobRunDetails); if (jobPunctualityDetails) { const { windowStartTime, windowEndTime } = jobPunctualityDetails; if ( diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js index 6c70b41d..a96b9e79 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js @@ -195,11 +195,6 @@ const Integrations = models.integrations; } } - console.log('------------------------------------------'); - console.dir("HERE") - console.log(newWorkUnitsFound); - console.log('------------------------------------------'); - // If no new monitoring work units are found, update the monitoring logs and exit if (!newWorkUnitsFound) { // If failed to reach cluster, do not update last monitored time in monitoring logs From 427ea011f919fdb591dc9ab6155f3d3381afa301 Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Wed, 21 Aug 2024 16:00:48 -0400 Subject: [PATCH 09/10] Fixed issue with job puntuality --- Tombolo/server/jobSchedular/job-scheduler.js | 30 +-- .../jobSchedularMethods/jobMonitoring.js | 12 +- .../jobMonitoring/monitorJobPunctuality.js | 175 ++++++++++-------- 3 files changed, 118 insertions(+), 99 deletions(-) diff --git a/Tombolo/server/jobSchedular/job-scheduler.js b/Tombolo/server/jobSchedular/job-scheduler.js index e211498d..9d01edf1 100644 --- a/Tombolo/server/jobSchedular/job-scheduler.js +++ b/Tombolo/server/jobSchedular/job-scheduler.js @@ -116,23 +116,23 @@ class JobScheduler { bootstrap() { (async () => { - await this.scheduleActiveCronJobs(); - await this.scheduleJobStatusPolling(); - await this.scheduleClusterTimezoneOffset(); - await this.scheduleFileMonitoring(); // file monitoring with templates - old file monitoring implementation - await this.scheduleFileMonitoringOnServerStart(); - await this.scheduleSuperFileMonitoringOnServerStart(); - await this.scheduleClusterMonitoringOnServerStart(); - await this.scheduleKeyCheck(); - await this.createClusterUsageHistoryJob(); + // await this.scheduleActiveCronJobs(); + // await this.scheduleJobStatusPolling(); + // await this.scheduleClusterTimezoneOffset(); + // await this.scheduleFileMonitoring(); // file monitoring with templates - old file monitoring implementation + // await this.scheduleFileMonitoringOnServerStart(); + // await this.scheduleSuperFileMonitoringOnServerStart(); + // await this.scheduleClusterMonitoringOnServerStart(); + // await this.scheduleKeyCheck(); + // await this.createClusterUsageHistoryJob(); await this.scheduleEmailNotificationProcessing(); - await this.scheduleTeamsNotificationProcessing(); - await this.scheduleOrbitMonitoringOnServerStart(); - await this.createOrbitMegaphoneJob(); - await this.startJobMonitoring(); - await this.startIntermediateJobsMonitoring(); + // await this.scheduleTeamsNotificationProcessing(); + // await this.scheduleOrbitMonitoringOnServerStart(); + // await this.createOrbitMegaphoneJob(); + // await this.startJobMonitoring(); + // await this.startIntermediateJobsMonitoring(); await this.startJobPunctualityMonitoring(); - await this.checkClusterReachability(); + // await this.checkClusterReachability(); })(); } diff --git a/Tombolo/server/jobSchedularMethods/jobMonitoring.js b/Tombolo/server/jobSchedularMethods/jobMonitoring.js index fb354615..ef743c58 100644 --- a/Tombolo/server/jobSchedularMethods/jobMonitoring.js +++ b/Tombolo/server/jobSchedularMethods/jobMonitoring.js @@ -26,8 +26,8 @@ async function startJobMonitoring() { let jobName = "job-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - interval: "10s", // For development - // interval: humanReadableIntervalForJobMonitoring, + // interval: "10s", // For development + interval: humanReadableIntervalForJobMonitoring, path: path.join( __dirname, "..", @@ -65,8 +65,8 @@ async function startIntermediateJobsMonitoring() { let jobName = "intermediate-state-jobs-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - interval: "20s", // For development - // interval: humanReadableIntervalForIntermediateJobMonitoring, + // interval: "20s", // For development + interval: humanReadableIntervalForIntermediateJobMonitoring, path: path.join( __dirname, "..", @@ -103,8 +103,8 @@ async function startJobPunctualityMonitoring() { let jobName = "job-punctuality-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - interval: "10s", // For development - // interval: humanReadableIntervalForJobPunctualityMonitoring, + // interval: "10s", // For development + interval: humanReadableIntervalForJobPunctualityMonitoring, path: path.join( __dirname, "..", diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js index 304ee730..7f59a29c 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js @@ -38,8 +38,16 @@ const Integrations = models.integrations; return; } - // All clusters - const clusters = await Cluster.findAll({ raw: true }); + // Get all unique clusters for the job monitorings + const clusterIds = jobMonitorings.map( + (jobMonitoring) => jobMonitoring.clusterId + ); + + // All clusters that are associated with the job monitorings + const clusters = await Cluster.findAll({ + where: { id: clusterIds }, + raw: true, + }); // Decrypt cluster passwords if they exist clusters.forEach((clusterInfo) => { @@ -95,76 +103,71 @@ const Integrations = models.integrations; continue; } - const { schedule, expectedStartTime, expectedCompletionTime } =metaData; + const { schedule, expectedStartTime, expectedCompletionTime } = metaData; const clusterInfo = clustersObj[clusterId]; - // Find severity level (For ASR ) - based on that determine when to send out notifications + // Find severity level (For ASR ) - based on that determine when to send out notifications let severityThreshHold = 0; let severeEmailRecipients = null; - if(metaData.asrSpecificMetaData){ - try{ - const {id : integrationId} = await Integrations.findOne({where: {name: "ASR"}, raw: true}); - - if(integrationId){ - // Get integration mapping with integration details - const integrationMapping = await IntegrationMapping.findOne({ - where: { - integration_id: integrationId, - application_id: applicationId, - }, + if (metaData.asrSpecificMetaData) { + try { + const { id: integrationId } = await Integrations.findOne({ + where: { name: "ASR" }, raw: true, }); - if(integrationMapping){ - const { - metaData: { - nocAlerts: { severityLevelForNocAlerts, emailContacts }, + if (integrationId) { + // Get integration mapping with integration details + const integrationMapping = await IntegrationMapping.findOne({ + where: { + integration_id: integrationId, + application_id: applicationId, }, - } = integrationMapping; - severityThreshHold = severityLevelForNocAlerts; - severeEmailRecipients = emailContacts; - } + raw: true, + }); + + if (integrationMapping) { + const { + metaData: { + nocAlerts: { severityLevelForNocAlerts, emailContacts }, + }, + } = integrationMapping; + severityThreshHold = severityLevelForNocAlerts; + severeEmailRecipients = emailContacts; + } } - }catch(error){ - logger.error(`Job Punctuality Monitoring : Error while getting integration level severity threshold: ${error.message}`); + } catch (error) { + logger.error( + `Job Punctuality Monitoring : Error while getting integration level severity threshold: ${error.message}` + ); } } - // Job level severity threshold const jobLevelSeverity = asrSpecificMetaData?.severity || 0; - // console.log('---------- SEVERITY ----------------------'); - // console.log(jobLevelSeverity, severityThreshHold) - // console.log('------------------------------------------'); - - - // If job level severity is less than the threshold, check only after the completion time + // Back date in minutes need to be calculated so run window is correctly calculated . EX - for overnight jobs let backDateInMs = 0; - if ( jobLevelSeverity < severityThreshHold) { - let runWindowForJob = null; - if (schedule[0]?.runWindow) { - runWindowForJob = schedule[0].runWindow; - } - - // Calculate the back date in ms - if (runWindowForJob === "overnight") { - backDateInMs = differenceInMs({ - startTime: expectedCompletionTime, - endTime: expectedStartTime, - daysDifference: 1, - }); - } else { - backDateInMs = differenceInMs({ - startTime: expectedCompletionTime, - endTime: expectedStartTime, - daysDifference: 0, - }); - } + let runWindowForJob = null; + if (schedule[0]?.runWindow) { + runWindowForJob = schedule[0].runWindow; } - console.log("-------------------------------------->") + // Calculate the back date in ms + if (runWindowForJob === "overnight") { + backDateInMs = differenceInMs({ + startTime: expectedCompletionTime, + endTime: expectedStartTime, + daysDifference: 1, + }); + } else { + backDateInMs = differenceInMs({ + startTime: expectedCompletionTime, + endTime: expectedStartTime, + daysDifference: 0, + }); + } // Calculate the run window for the job const window = calculateRunOrCompleteByTimes({ @@ -175,45 +178,53 @@ const Integrations = models.integrations; backDateInMs, }); - // If the window null - continue. Job is not expected to run + // If the window null - continue. Job is not expected to run if (!window) { continue; } let alertTimePassed = false; - let lateByInMinutes = 0; - if(jobLevelSeverity < severityThreshHold){ + let lateByInMinutes = 0; + + /* jobLevelSeverity < severityThreshHold means the job is not so severe. + We can wait until expected completion time before notifying about unpunctuality */ + + if (jobLevelSeverity < severityThreshHold || !severityThreshHold) { alertTimePassed = window.end < window.currentTime; - lateByInMinutes = Math.floor((window.currentTime - window.end) / 60000); - }else{ + lateByInMinutes = Math.floor( + (window.currentTime - window.end) / 60000 + ); + } else { alertTimePassed = window.start < window.currentTime; - lateByInMinutes = Math.floor((window.currentTime - window.start) / 60000); + lateByInMinutes = Math.floor( + (window.currentTime - window.start) / 60000 + ); } - // Give grace period of 10 minutes - if(lateByInMinutes >10){ - timePassed = true; - } - - // If the time has not passed, continue - if (!alertTimePassed) { + // If the time has not passed, or with in grace period of 10 minutes, continue + if (!alertTimePassed || lateByInMinutes < 10) { continue; } // Check if notification has been sent out for this job, for the current window const jobPunctualityDetails = lastJobRunDetails?.jobPunctualityDetails; - console.log("------", lastJobRunDetails); + let notificationAlreadySentForThisWindow = false; + if (jobPunctualityDetails) { const { windowStartTime, windowEndTime } = jobPunctualityDetails; if ( windowStartTime === window.start.toISOString() && windowEndTime === window.end.toISOString() ) { - continue; + notificationAlreadySentForThisWindow = true; } } + // If notification has already been sent for this window for this JM- continue + if (notificationAlreadySentForThisWindow) { + continue; + } // Make a call to HPCC to see if the job has started const translatedJobName = generateJobName({ @@ -238,7 +249,7 @@ const Integrations = models.integrations; }); // If a job is overnight, it could potentially have 2 translatedJobName as it can run on 2 different days - if(schedule[0]?.runWindow === 'overnight'){ + if (schedule[0]?.runWindow === "overnight") { const translatedJobNameNextDay = generateJobName({ pattern: jobNamePattern, timezone_offset: clusterInfo.timezone_offset, @@ -305,7 +316,7 @@ const Integrations = models.integrations; const notificationPayload = createNotificationPayload({ type: "email", notificationDescription: - "Monitoring detected that a monitored job has not started on time", + "Monitoring detected that a monitored job did not started on time", templateName: "jobMonitoring", originationId: monitoringTypeDetails.id, applicationId: applicationId, @@ -335,8 +346,12 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date(now.getTime() + offSet * 60 * 1000).toISOString(), - lastLogged: new Date(now.getTime() + offSet * 60 * 1000).toISOString(), + firstLogged: new Date( + now.getTime() + offSet * 60 * 1000 + ).toISOString(), + lastLogged: new Date( + now.getTime() + offSet * 60 * 1000 + ).toISOString(), }); // Queue email notification @@ -345,8 +360,10 @@ const Integrations = models.integrations; // NOC email notification if (jobLevelSeverity >= severityThreshHold && severeEmailRecipients) { const notificationPayloadForNoc = { ...notificationPayload }; - notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; - notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayloadForNoc.metaData.notificationDescription = + nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = + severeEmailRecipients; notificationPayloadForNoc.metaData.notificationId = generateNotificationId({ notificationPrefix, @@ -377,12 +394,14 @@ const Integrations = models.integrations; logger.error( `Error while processing jobs for punctuality check ${jobMonitoring.id}: ${error.message}` ); - }finally{ - if (parentPort) parentPort.postMessage("done"); - else process.exit(0); } } } catch (error) { - logger.error(`Error in job punctuality monitoring script: ${error.message}`); + logger.error( + `Error in job punctuality monitoring script: ${error.message}` + ); + } finally { + if (parentPort) parentPort.postMessage("done"); + else process.exit(0); } })(); \ No newline at end of file From 0767264958cef5de1b34bcaff112e88cf628a425 Mon Sep 17 00:00:00 2001 From: yadhap Dahal Date: Wed, 21 Aug 2024 16:15:51 -0400 Subject: [PATCH 10/10] Remvoed unused console.logs and uncommented/commented unused code --- Tombolo/server/config/logger.js | 10 +++---- Tombolo/server/jobSchedular/job-scheduler.js | 30 +++++++++---------- .../processEmailNotifications.js | 4 --- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/Tombolo/server/config/logger.js b/Tombolo/server/config/logger.js index 8ddf8ea1..e8e81f5f 100644 --- a/Tombolo/server/config/logger.js +++ b/Tombolo/server/config/logger.js @@ -49,10 +49,10 @@ const logger = createLogger({ }); // If we're not in production then also log to the files -if (process.env.NODE_ENV !== 'production') { - const settings = {...common, format: getFormat()}; // will write to files same output as to console but no special char for coloring - logger.add(new transports.File({ ...settings, level: 'http', filename: './logs/combined.log' })); - logger.add(new transports.File({ ...settings, level: 'error', filename: './logs/error.log' })); // only logger.error() will be written here -} +// if (process.env.NODE_ENV !== 'production') { +// const settings = {...common, format: getFormat()}; // will write to files same output as to console but no special char for coloring +// logger.add(new transports.File({ ...settings, level: 'http', filename: './logs/combined.log' })); +// logger.add(new transports.File({ ...settings, level: 'error', filename: './logs/error.log' })); // only logger.error() will be written here +// } module.exports = logger; \ No newline at end of file diff --git a/Tombolo/server/jobSchedular/job-scheduler.js b/Tombolo/server/jobSchedular/job-scheduler.js index 9d01edf1..e211498d 100644 --- a/Tombolo/server/jobSchedular/job-scheduler.js +++ b/Tombolo/server/jobSchedular/job-scheduler.js @@ -116,23 +116,23 @@ class JobScheduler { bootstrap() { (async () => { - // await this.scheduleActiveCronJobs(); - // await this.scheduleJobStatusPolling(); - // await this.scheduleClusterTimezoneOffset(); - // await this.scheduleFileMonitoring(); // file monitoring with templates - old file monitoring implementation - // await this.scheduleFileMonitoringOnServerStart(); - // await this.scheduleSuperFileMonitoringOnServerStart(); - // await this.scheduleClusterMonitoringOnServerStart(); - // await this.scheduleKeyCheck(); - // await this.createClusterUsageHistoryJob(); + await this.scheduleActiveCronJobs(); + await this.scheduleJobStatusPolling(); + await this.scheduleClusterTimezoneOffset(); + await this.scheduleFileMonitoring(); // file monitoring with templates - old file monitoring implementation + await this.scheduleFileMonitoringOnServerStart(); + await this.scheduleSuperFileMonitoringOnServerStart(); + await this.scheduleClusterMonitoringOnServerStart(); + await this.scheduleKeyCheck(); + await this.createClusterUsageHistoryJob(); await this.scheduleEmailNotificationProcessing(); - // await this.scheduleTeamsNotificationProcessing(); - // await this.scheduleOrbitMonitoringOnServerStart(); - // await this.createOrbitMegaphoneJob(); - // await this.startJobMonitoring(); - // await this.startIntermediateJobsMonitoring(); + await this.scheduleTeamsNotificationProcessing(); + await this.scheduleOrbitMonitoringOnServerStart(); + await this.createOrbitMegaphoneJob(); + await this.startJobMonitoring(); + await this.startIntermediateJobsMonitoring(); await this.startJobPunctualityMonitoring(); - // await this.checkClusterReachability(); + await this.checkClusterReachability(); })(); } diff --git a/Tombolo/server/jobs/notifications/processEmailNotifications.js b/Tombolo/server/jobs/notifications/processEmailNotifications.js index bb7e58e6..9e131874 100644 --- a/Tombolo/server/jobs/notifications/processEmailNotifications.js +++ b/Tombolo/server/jobs/notifications/processEmailNotifications.js @@ -98,10 +98,6 @@ const SentNotification = models.sent_notifications; // Send email const emailResponse = await sendEmail({ ...emailPayload }); - console.log('------------------------------------------'); - console.dir(emailResponse) - console.log('------------------------------------------'); - // Assume success - if no error is thrown successfulDelivery.push(emailPayload); }