diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx index 09682fcb..4ba9657b 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringFilters.jsx @@ -35,7 +35,7 @@ function NotificationTableFilters({ const [domainOptions, setDomainOptions] = useState([]); const [productOptions, setProductOptions] = useState([]); const [frequencyOptions, setFrequencyOptions] = useState([]); - const [filterCount, setFilterCount] = useState(4); + const [filterCount, setFilterCount] = useState(0); //Effects useEffect(() => { @@ -151,6 +151,17 @@ function NotificationTableFilters({ setFiltersVisible(true); }; + // Clear filters when clear is clicked + const clearFilters = () => { + form.resetFields(); + setFilterCount(0); + setFilters({}); + // If exists remove jMFilters from local storage + if (localStorage.getItem('jMFilters')) { + localStorage.removeItem('jMFilters'); + } + }; + //JSX return (
@@ -234,9 +245,15 @@ function NotificationTableFilters({ {filterCount > 0 && !filtersVisible && (
-
+
{`${filterCount} filter(s) active`} - - View + + - View + + + {' '} + | Clear +
)} diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx index ec92cb00..2e9c7455 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/JobMonitoringTable.jsx @@ -39,6 +39,7 @@ const JobMonitoringTable = ({ setSelectedRows, domains, allProductCategories, + filteringJobs, }) => { //Redux const { @@ -279,6 +280,7 @@ const JobMonitoringTable = ({ return ( )} - {asrSpecificMetaData?.severity && ( + {asrSpecificMetaData?.severity !== undefined && asrSpecificMetaData?.severity !== null && ( {asrSpecificMetaData.severity} )} {/* ---------NOTIFICATION TRIGGERS AND CONTACTS --------------------------------------------- */} @@ -213,22 +213,33 @@ function MonitoringDetailsModal({ export default MonitoringDetailsModal; +// Interpret run window +const interpretRunWindow = (schedule) => { + const runWindow = schedule[0].runWindow || ''; + if (runWindow === '') { + return ''; + } else if (runWindow === 'daily') return 'Anytime'; + else { + return _.capitalize(runWindow); + } +}; + //Generate tags for schedule const generateTagsForSchedule = (schedule) => { const tags = []; schedule.forEach((s) => { if (s.frequency === 'daily') { - tags.push('Everyday'); + tags.push(interpretRunWindow(schedule)); } if (s.frequency === 'weekly') { - let tempData = 'Every week on'; + let tempData = `Every Week ${interpretRunWindow(schedule)} on`; s.days.forEach((d, i) => { tempData += ` ${getDayLabel(d)} ${i < s.days.length - 1 ? ',' : ''}`; }); tags.push(tempData); } if (s.scheduleBy === 'dates') { - let tempData = 'Every month on'; + let tempData = `Every month ${interpretRunWindow(schedule)}`; s.dates.forEach((d, i) => { tempData += ` ${getDateLabel(d)} ${i < s.dates.length - 1 ? ',' : ''}`; }); diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx index 7bb93284..01eaaca3 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/SchedulePicker.jsx @@ -10,7 +10,7 @@ const dailyRunWindowAndIntervals = [ { label: 'Morning (00:00 - 11:59)', value: 'morning' }, { label: 'Afternoon (12:00 - 23:59)', value: 'afternoon' }, { label: 'Overnight (Prev 12:00 - Current day 12:00)', value: 'overnight' }, - { label: 'Every 2 Days', value: 'every2Days' }, + // { label: 'Every 2 Days', value: 'every2Days' }, ]; // Daily schedule options diff --git a/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx b/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx index 7986503b..3c215466 100644 --- a/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx +++ b/Tombolo/client-reactjs/src/components/application/jobMonitoring/index.jsx @@ -73,6 +73,7 @@ function JobMonitoring() { const [bulkEditModalVisibility, setBulkEditModalVisibility] = useState(false); const [filters, setFilters] = useState({}); const [filtersVisible, setFiltersVisible] = useState(true); + const [filteringJobs, setFilteringJobs] = useState(false); // Create form instance const [form] = Form.useForm(); @@ -84,7 +85,6 @@ function JobMonitoring() { try { const allMonitorings = await getAllJobMonitorings({ applicationId }); setJobMonitorings(allMonitorings); - setFilteredJobMonitoring(allMonitorings); } catch (error) { message.error('Error fetching job monitorings'); } @@ -138,7 +138,7 @@ function JobMonitoring() { } }, [editingData, duplicatingData]); - // Get all teams hook, monitoring type ID, Filters from local storage + // Get monitoring type ID, Filters from local storage useEffect(() => { // Get monitoringType id for job monitoring (async () => { @@ -198,8 +198,11 @@ function JobMonitoring() { // When filterChange filter the job monitorings useEffect(() => { - if (jobMonitorings.length === 0) return; - if (Object.keys(filters).length < 1) return; + setFilteringJobs(true); + if (jobMonitorings.length === 0) { + setFilteringJobs(false); + } + // if (Object.keys(filters).length < 1) return; const { approvalStatus, activeStatus, domain, frequency, product } = filters; // Convert activeStatus to boolean @@ -210,7 +213,7 @@ function JobMonitoring() { activeStatusBool = false; } - const filteredJobMonitorings = jobMonitorings.filter((jobMonitoring) => { + const filteredJm = jobMonitorings.filter((jobMonitoring) => { let include = true; const currentDomain = jobMonitoring?.metaData?.asrSpecificMetaData?.domain; const currentProduct = jobMonitoring?.metaData?.asrSpecificMetaData?.productCategory; @@ -237,7 +240,8 @@ function JobMonitoring() { return include; }); - setFilteredJobMonitoring(filteredJobMonitorings); + setFilteredJobMonitoring(filteredJm); + setFilteringJobs(false); }, [filters, jobMonitorings]); // Function reset states when modal is closed @@ -385,6 +389,7 @@ function JobMonitoring() { allInputs = { ...allInputs, metaData }; const responseData = await createJobMonitoring({ inputData: allInputs }); + setJobMonitorings([responseData, ...jobMonitorings]); message.success('Job monitoring saved successfully'); @@ -648,6 +653,7 @@ function JobMonitoring() { domains={domains} productCategories={productCategories} allProductCategories={allProductCategories} + filteringJobs={filteringJobs} /> { const response = await fetch(`/api/jobmonitoring`, payload); if (!response.ok) { - return message.error('Failed to save job monitoring'); + throw new Error('Failed to save job monitoring'); } const data = await response.json(); @@ -76,7 +76,7 @@ export const updateSelectedMonitoring = async ({ updatedData }) => { const response = await fetch(`/api/jobmonitoring/`, payload); if (!response.ok) { - return message.error('Failed to update job monitoring'); + throw new Error('Failed to update job monitoring'); } const data = await response.json(); @@ -94,7 +94,7 @@ export const handleDeleteJobMonitoring = async ({ id, jobMonitorings, setJobMoni const response = await fetch(`/api/jobmonitoring/${id}`, payload); if (!response.ok) { - return message.error('Failed to delete job monitoring'); + throw new Error('Failed to delete job monitoring'); } // Set job monitorings @@ -102,7 +102,7 @@ export const handleDeleteJobMonitoring = async ({ id, jobMonitorings, setJobMoni setJobMonitorings(filteredJobMonitorings); message.success('Job monitoring deleted successfully'); } catch (err) { - message.error(err.message); + throw new Error(err.message); } }; diff --git a/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx b/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx index d0173e2b..a0f8770b 100644 --- a/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx +++ b/Tombolo/client-reactjs/src/components/userGuides/JobNamePattern.jsx @@ -106,8 +106,9 @@ function JobNamePattern() {

Invalid Job Name

- If an invalid job name is provided, the system will handle it in the same way as an invalid host name. Please - see the Host field notes for details. + If an invalid job name or pattern is provided, Tombolo won't be able to monitor the job correctly. It may + search for a non-existent name or mistakenly monitor a wrong job that matches the pattern. Therefore, please + ensure the name or pattern you provide is correct.

); diff --git a/Tombolo/server/jobSchedularMethods/jobMonitoring.js b/Tombolo/server/jobSchedularMethods/jobMonitoring.js index 20f80fce..ef743c58 100644 --- a/Tombolo/server/jobSchedularMethods/jobMonitoring.js +++ b/Tombolo/server/jobSchedularMethods/jobMonitoring.js @@ -26,7 +26,7 @@ async function startJobMonitoring() { let jobName = "job-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - //interval: "10s", // For development + // interval: "10s", // For development interval: humanReadableIntervalForJobMonitoring, path: path.join( __dirname, @@ -65,7 +65,7 @@ async function startIntermediateJobsMonitoring() { let jobName = "intermediate-state-jobs-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - //interval: "20s", // For development + // interval: "20s", // For development interval: humanReadableIntervalForIntermediateJobMonitoring, path: path.join( __dirname, @@ -103,7 +103,7 @@ async function startJobPunctualityMonitoring() { let jobName = "job-punctuality-monitoring" + new Date().getTime(); this.bree.add({ name: jobName, - //interval: "10s", // For development + // interval: "10s", // For development interval: humanReadableIntervalForJobPunctualityMonitoring, path: path.join( __dirname, diff --git a/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js index ec259136..8309a447 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorIntermediateStateJobs.js @@ -16,6 +16,7 @@ const { getProductCategory, getDomain, createNotificationPayload, + nocAlertDescription, } = require("./monitorJobsUtil"); // Constants @@ -74,7 +75,7 @@ const Integrations = models.integrations; clusterInfo.localTime = findLocalDateTimeAtCluster( clusterInfo.timezone_offset || 0 - ).toISOString(); + ); } catch (error) { logger.error( `Failed to decrypt hash for cluster ${clusterInfo.id}: ${error.message}` @@ -105,7 +106,7 @@ const Integrations = models.integrations; // Find severity level (For ASR ) - based on that determine when to send out notifications let severityThreshHold = 0; - let severeEmailRecipients; + let severeEmailRecipients = null; try { const { id: integrationId } = await Integrations.findOne({ where: { name: "ASR" }, @@ -119,17 +120,19 @@ const Integrations = models.integrations; raw: true, }); - const { - metaData: { - nocAlerts: { severityLevelForNocAlerts, emailContacts }, - }, - } = integrationMapping; - severityThreshHold = severityLevelForNocAlerts; - severeEmailRecipients = emailContacts; + if(integrationMapping){ + const { + metaData: { + nocAlerts: { severityLevelForNocAlerts, emailContacts }, + }, + } = integrationMapping; + severityThreshHold = severityLevelForNocAlerts; + severeEmailRecipients = emailContacts; + } } } catch (error) { logger.error( - `Job Monitoring : Error while getting integration level severity threshold: ${error.message}` + `Intermediate State Job Monitoring : Error while getting integration level severity threshold: ${error.message}` ); } @@ -213,9 +216,7 @@ const Integrations = models.integrations; try { const info = await wuService.WUInfo({ Wuid: wu.Wuid }); - const { - Workunit: { State }, - } = info; + const {Workunit: { State }} = info; // Check if current time is before, after, within the window const currentTimeToWindowRelation = @@ -224,9 +225,16 @@ const Integrations = models.integrations; end: expectedCompletionTime, currentTime: clusterDetail.localTime, }); + + + const sendAlertToNoc = severity >= severityThreshHold && severeEmailRecipients; + // WU now in state such as failed, aborted etc if (notificationConditionLowerCase.includes(State)) { + console.log('------------------------------------------'); + console.dir("Intermediate job failed") + console.log('------------------------------------------'); // Add new State to the WU wu.State = _.capitalize(State); @@ -252,10 +260,9 @@ const Integrations = models.integrations; "Job Name/Filter": jobNamePattern, "Returned Job": wu.Jobname, State: wu.State, - "Discovered at": - new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString() || now.toISOString(), + "Discovered at": findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }, notificationId: generateNotificationId({ notificationPrefix, @@ -267,29 +274,28 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), - lastLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), + firstLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), + lastLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }); - // notificationPayload.wuId = wu.Wuid; + notificationPayload.wuId = wu.Wuid; notificationsToBeQueued.push(notificationPayload); // NOC email notification if severity is high - if ( - severity >= severityThreshHold && - severeEmailRecipients - ) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; - notificationPayload.metaData.mainRecipients = - severeEmailRecipients; - delete notificationPayload.metaData.cc; - notificationsToBeQueued.push(notificationPayload); + if (sendAlertToNoc) { + const notificationPayloadForNoc = { ...notificationPayload} + notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayloadForNoc.metaData.notificationId = generateNotificationId({ + notificationPrefix, + timezoneOffset: clusterDetail.timezone_offset || 0, + }), + delete notificationPayloadForNoc.metaData.cc; + notificationsToBeQueued.push(notificationPayloadForNoc); } wuNoLongerInIntermediateState.push(wu.Wuid); @@ -300,6 +306,10 @@ const Integrations = models.integrations; currentTimeToWindowRelation === "after" && requireComplete === true ) { + console.log('------------------------------------------'); + console.dir("Intermediate job passed time") + console.log('------------------------------------------'); + // Add new State to the WU wu.State = _.capitalize(State); @@ -325,10 +335,9 @@ const Integrations = models.integrations; "Job Name/Filter": jobNamePattern, "Returned Job": wu.Jobname, State: wu.State, - "Discovered at": - new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString() || now.toISOString(), + "Discovered at": findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }, notificationId: generateNotificationId({ notificationPrefix, @@ -340,27 +349,42 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), - lastLogged: new Date( - now + clusterDetail.timezone_offset * 60 * 1000 - ).toISOString(), + firstLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), + lastLogged: findLocalDateTimeAtCluster( + clusterDetail.timezone_offset + ), }); + + // console.log('-----------Payload 1----------------------'); + // console.dir(notificationPayload) + // console.log('------------------------------------------'); notificationsToBeQueued.push(notificationPayload); // NOC email notification if severity is high - if (severity >= severityThreshHold && severeEmailRecipients) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; - notificationPayload.metaData.mainRecipients = - severeEmailRecipients; - delete notificationPayload.metaData.cc; - notificationsToBeQueued.push(notificationPayload); + if (sendAlertToNoc) { + const notificationPayloadForNoc = { ...notificationPayload} + notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayloadForNoc.metaData.notificationId = generateNotificationId({ + notificationPrefix, + timezoneOffset: clusterDetail.timezone_offset || 0, + }), + delete notificationPayloadForNoc.metaData.cc; + + + // console.log("-----------Payload 2----------------------"); + // console.dir(notificationPayload); + // console.log("------------------------------------------"); + + notificationsToBeQueued.push(notificationPayloadForNoc); } - + console.log('----------To be queued -------------------'); + console.dir(notificationsToBeQueued) + console.log('------------------------------------------'); + wuNoLongerInIntermediateState.push(wu.Wuid); } // IF the job is still in intermediate state and the current time is within the run window @@ -368,17 +392,24 @@ const Integrations = models.integrations; intermediateStates.includes(State) && currentTimeToWindowRelation === "within" ) { + console.log('------------------------------------------'); + console.dir("STILL IN INTERMEDIATE STATE") + console.log('------------------------------------------'); + // If the State has changed from last time it was checked, update monitoring needs to be updated with new state if (wu.State !== State) { wuWithNewIntermediateState[wu.Wuid] = State; } } else { + console.log('------------------------------------------'); + console.dir("COMPLETED") + console.log('------------------------------------------'); // WU in completed state - Remove the WU from the intermediate state wuNoLongerInIntermediateState.push(wu.Wuid); } } catch (err) { logger.error( - `WUId - ${wu.Wuid} - Cluster ${cluster_id}: ${err.message}` + `Monitor Intermediate Jobs. WUId - ${wu.Wuid} - Cluster ${cluster_id}: ${err.message}` ); } } @@ -389,6 +420,9 @@ const Integrations = models.integrations; // Insert notification in queue for (let notification of notificationsToBeQueued) { + // console.log('-NOTIFICATION LOOP ------------------------'); + // console.dir(notification); + // console.log('------------------------------------------'); await notification_queue.create(notification); } @@ -429,15 +463,12 @@ const Integrations = models.integrations; { where: { id } } ); } catch (error) { - logger.error(`Error updating log with id ${log.id}:`, error); + logger.error(`Intermediate State Jobs - Error updating log with id ${log.id}:`, error); } } } catch (err) { logger.error(err); } finally { - logger.debug( - `Job monitoring completed started ${now} and ended at ${new Date()}` - ); if (parentPort) parentPort.postMessage("done"); else process.exit(0); } diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js index 2583ecd4..7f59a29c 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobPunctuality.js @@ -1,6 +1,7 @@ const { WorkunitsService } = require("@hpcc-js/comms"); const logger = require("../../config/logger"); const { decryptString } = require("../../utils/cipher"); +const { parentPort } = require("worker_threads"); const { calculateRunOrCompleteByTimes, generateJobName, @@ -8,6 +9,8 @@ const { getProductCategory, getDomain, generateNotificationId, + differenceInMs, + nocAlertDescription, } = require("./monitorJobsUtil"); const models = require("../../models"); @@ -24,7 +27,6 @@ const Integrations = models.integrations; const now = new Date(); // UTC time try { - // Get all job monitorings // Find all active job monitorings. const jobMonitorings = await JobMonitoring.findAll({ where: { isActive: 1, approvalStatus: "Approved" }, @@ -36,33 +38,16 @@ const Integrations = models.integrations; return; } - // Find severity level (For ASR ) - based on that determine when to send out notifications - let severityThreshHold = 0; - let severeEmailRecipients; - - try{ - const {id : integrationId} = await Integrations.findOne({where: {name: "ASR"}, raw: true}); - - if(integrationId){ - // Get integration mapping with integration details - const integrationMapping = await IntegrationMapping.findOne({ - where : {integration_id : integrationId}, raw: true - }); - - const { - metaData: { - nocAlerts: { severityLevelForNocAlerts, emailContacts }, - }, - } = integrationMapping; - severityThreshHold = severityLevelForNocAlerts; - severeEmailRecipients = emailContacts; - } - }catch(error){ - logger.error(`Job Punctuality Monitoring : Error while getting integration level severity threshold: ${error.message}`); - } + // Get all unique clusters for the job monitorings + const clusterIds = jobMonitorings.map( + (jobMonitoring) => jobMonitoring.clusterId + ); - // All clusters - const clusters = await Cluster.findAll({ raw: true }); + // All clusters that are associated with the job monitorings + const clusters = await Cluster.findAll({ + where: { id: clusterIds }, + raw: true, + }); // Decrypt cluster passwords if they exist clusters.forEach((clusterInfo) => { @@ -118,16 +103,70 @@ const Integrations = models.integrations; continue; } - const { schedule, expectedStartTime, expectedCompletionTime } =metaData; + const { schedule, expectedStartTime, expectedCompletionTime } = metaData; const clusterInfo = clustersObj[clusterId]; + // Find severity level (For ASR ) - based on that determine when to send out notifications + let severityThreshHold = 0; + let severeEmailRecipients = null; + + if (metaData.asrSpecificMetaData) { + try { + const { id: integrationId } = await Integrations.findOne({ + where: { name: "ASR" }, + raw: true, + }); + + if (integrationId) { + // Get integration mapping with integration details + const integrationMapping = await IntegrationMapping.findOne({ + where: { + integration_id: integrationId, + application_id: applicationId, + }, + raw: true, + }); + + if (integrationMapping) { + const { + metaData: { + nocAlerts: { severityLevelForNocAlerts, emailContacts }, + }, + } = integrationMapping; + severityThreshHold = severityLevelForNocAlerts; + severeEmailRecipients = emailContacts; + } + } + } catch (error) { + logger.error( + `Job Punctuality Monitoring : Error while getting integration level severity threshold: ${error.message}` + ); + } + } + // Job level severity threshold const jobLevelSeverity = asrSpecificMetaData?.severity || 0; - // If job level severity is less than the threshold, check only after the completion time - let backDateInMinutes = 0; - if(jobLevelSeverity < severityThreshHold){ - backDateInMinutes = 1440; // 24 hours + // Back date in minutes need to be calculated so run window is correctly calculated . EX - for overnight jobs + let backDateInMs = 0; + let runWindowForJob = null; + if (schedule[0]?.runWindow) { + runWindowForJob = schedule[0].runWindow; + } + + // Calculate the back date in ms + if (runWindowForJob === "overnight") { + backDateInMs = differenceInMs({ + startTime: expectedCompletionTime, + endTime: expectedStartTime, + daysDifference: 1, + }); + } else { + backDateInMs = differenceInMs({ + startTime: expectedCompletionTime, + endTime: expectedStartTime, + daysDifference: 0, + }); } // Calculate the run window for the job @@ -136,45 +175,56 @@ const Integrations = models.integrations; timezone_offset: clusterInfo.timezone_offset, expectedStartTime, expectedCompletionTime, - backDateInMinutes, + backDateInMs, }); - // If the window null - continue. Job is not expected to run + // If the window null - continue. Job is not expected to run if (!window) { continue; } - let timePassed = false; - let lateByInMinutes = 0; - if(jobLevelSeverity < severityThreshHold){ - lateByInMinutes = Math.floor((window.currentTime - window.end) / 60000); - }else{ - timePassed = window.start < window.currentTime; - lateByInMinutes = Math.floor((window.currentTime - window.start) / 60000); - } + let alertTimePassed = false; + let lateByInMinutes = 0; - // Give grace period of 10 minutes - if(lateByInMinutes >10){ - timePassed = true; + /* jobLevelSeverity < severityThreshHold means the job is not so severe. + We can wait until expected completion time before notifying about unpunctuality */ + + if (jobLevelSeverity < severityThreshHold || !severityThreshHold) { + alertTimePassed = window.end < window.currentTime; + lateByInMinutes = Math.floor( + (window.currentTime - window.end) / 60000 + ); + } else { + alertTimePassed = window.start < window.currentTime; + lateByInMinutes = Math.floor( + (window.currentTime - window.start) / 60000 + ); } - - // If the time has not passed, continue - if (!timePassed) { + + // If the time has not passed, or with in grace period of 10 minutes, continue + if (!alertTimePassed || lateByInMinutes < 10) { continue; } // Check if notification has been sent out for this job, for the current window const jobPunctualityDetails = lastJobRunDetails?.jobPunctualityDetails; + + let notificationAlreadySentForThisWindow = false; + if (jobPunctualityDetails) { const { windowStartTime, windowEndTime } = jobPunctualityDetails; if ( windowStartTime === window.start.toISOString() && windowEndTime === window.end.toISOString() ) { - continue; + notificationAlreadySentForThisWindow = true; } } + // If notification has already been sent for this window for this JM- continue + if (notificationAlreadySentForThisWindow) { + continue; + } // Make a call to HPCC to see if the job has started const translatedJobName = generateJobName({ @@ -198,6 +248,25 @@ const Integrations = models.integrations; Jobname: translatedJobName, }); + // If a job is overnight, it could potentially have 2 translatedJobName as it can run on 2 different days + if (schedule[0]?.runWindow === "overnight") { + const translatedJobNameNextDay = generateJobName({ + pattern: jobNamePattern, + timezone_offset: clusterInfo.timezone_offset, + backDateInDays: 1, + }); + + const { + Workunits: { ECLWorkunit: ECLWorkunitNextDay }, + } = await wuService.WUQuery({ + StartDate: window.start, + EndDate: window.end, + Jobname: translatedJobNameNextDay, + }); + + ECLWorkunit.push(...ECLWorkunitNextDay); + } + // If workunits are found, update the job monitoring and continue if (ECLWorkunit.length > 0) { await JobMonitoring.update( @@ -247,7 +316,7 @@ const Integrations = models.integrations; const notificationPayload = createNotificationPayload({ type: "email", notificationDescription: - "Monitoring detected that a monitored job has not started on time", + "Monitoring detected that a monitored job did not started on time", templateName: "jobMonitoring", originationId: monitoringTypeDetails.id, applicationId: applicationId, @@ -277,8 +346,12 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date(now.getTime() + offSet * 60 * 1000).toISOString(), - lastLogged: new Date(now.getTime() + offSet * 60 * 1000).toISOString(), + firstLogged: new Date( + now.getTime() + offSet * 60 * 1000 + ).toISOString(), + lastLogged: new Date( + now.getTime() + offSet * 60 * 1000 + ).toISOString(), }); // Queue email notification @@ -286,12 +359,18 @@ const Integrations = models.integrations; // NOC email notification if (jobLevelSeverity >= severityThreshHold && severeEmailRecipients) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.` - notificationPayload.metaData.mainRecipients = severeEmailRecipients; - delete notificationPayload.metaData.cc; - await NotificationQueue.create(notificationPayload); + const notificationPayloadForNoc = { ...notificationPayload }; + notificationPayloadForNoc.metaData.notificationDescription = + nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = + severeEmailRecipients; + notificationPayloadForNoc.metaData.notificationId = + generateNotificationId({ + notificationPrefix, + timezoneOffset: offSet || 0, + }); + delete notificationPayloadForNoc.metaData.cc; + await NotificationQueue.create(notificationPayloadForNoc); } // Update the job monitoring await JobMonitoring.update( @@ -321,5 +400,8 @@ const Integrations = models.integrations; logger.error( `Error in job punctuality monitoring script: ${error.message}` ); + } finally { + if (parentPort) parentPort.postMessage("done"); + else process.exit(0); } })(); \ No newline at end of file diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js index 0a27a372..a96b9e79 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobs.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobs.js @@ -14,6 +14,8 @@ const { generateNotificationId, getProductCategory, getDomain, + findLocalDateTimeAtCluster, + nocAlertDescription, } = require("./monitorJobsUtil"); const e = require("express"); @@ -53,7 +55,7 @@ const Integrations = models.integrations; // Find severity level (For ASR ) - based on that determine when to send out notifications let severityThreshHold = 0; - let severeEmailRecipients; + let severeEmailRecipients = null; try { const { id: integrationId } = await Integrations.findOne({ @@ -68,13 +70,16 @@ const Integrations = models.integrations; raw: true, }); - const { - metaData: { - nocAlerts: { severityLevelForNocAlerts, emailContacts }, - }, - } = integrationMapping; - severityThreshHold = severityLevelForNocAlerts; - severeEmailRecipients = emailContacts; + + if(integrationMapping){ + const { + metaData: { + nocAlerts: { severityLevelForNocAlerts, emailContacts }, + }, + } = integrationMapping; + severityThreshHold = severityLevelForNocAlerts; + severeEmailRecipients = emailContacts; + } } } catch (error) { logger.error( @@ -154,6 +159,7 @@ const Integrations = models.integrations; /* Fetch basic information for all work units per cluster */ const wuBasicInfoByCluster = {}; + const failedToReachClusters = []; for (let clusterInfo of clustersInfo) { try { const wuService = new WorkunitsService({ @@ -174,9 +180,8 @@ const Integrations = models.integrations; wuBasicInfoByCluster[clusterInfo.id] = [...wuWithClusterIds]; } catch (err) { - logger.error( - `Job monitoring - Error while reaching out to cluster ${clusterInfo.id} : ${err}` - ); + failedToReachClusters.push(clusterInfo.id); + logger.error(`Job monitoring - Error while reaching out to cluster ${clusterInfo.id} : ${err}`); } } @@ -192,7 +197,12 @@ const Integrations = models.integrations; // If no new monitoring work units are found, update the monitoring logs and exit if (!newWorkUnitsFound) { - for (let id of clusterIds) { + // If failed to reach cluster, do not update last monitored time in monitoring logs + const scanned_clusters = clusterIds.filter( + (id) => !failedToReachClusters.includes(id) + ); + + for (let id of scanned_clusters) { // grab existing metaData const log = await MonitoringLogs.findOne({ where: { monitoring_type_id: monitoringTypeId, cluster_id: id }, @@ -371,10 +381,9 @@ const Integrations = models.integrations; Cluster: clusterInfoObj[clusterId].name || "", "Job Name/Filter": jobName, "Returned Job": wu.Jobname, - "Discovered at": - new Date( - now + clusterInfoObj[clusterId].timezone_offset * 60 * 1000 - ).toISOString() || now.toISOString(), + "Discovered at": findLocalDateTimeAtCluster( + clusterInfoObj[clusterId].timezone_offset + ), State: wu.State, }, notificationId: generateNotificationId({ @@ -387,12 +396,12 @@ const Integrations = models.integrations; domain, severity, }, // region: "USA", product: "Telematics", domain: "Insurance", severity: 3, - firstLogged: new Date( - now + clusterInfoObj[clusterId].timezone_offset * 60 * 1000 - ).toISOString(), - lastLogged: new Date( - now + clusterInfoObj[clusterId].timezone_offset * 60 * 1000 - ).toISOString(), + firstLogged: findLocalDateTimeAtCluster( + clusterInfoObj[clusterId].timezone_offset + ), + lastLogged: findLocalDateTimeAtCluster( + clusterInfoObj[clusterId].timezone_offset + ), }); //Create notification queue @@ -400,17 +409,25 @@ const Integrations = models.integrations; // If severity is above threshold, send out NOC notification if (severity >= severityThreshHold && severeEmailRecipients) { - notificationPayload.metaData.notificationDescription = `[SEV TICKET REQUEST] - The following issue has been identified via automation. - Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; - notificationPayload.metaData.mainRecipients = severeEmailRecipients; - delete notificationPayload.metaData.cc; - await NotificationQueue.create(notificationPayload); + const notificationPayloadForNoc = { ...notificationPayload }; + notificationPayloadForNoc.metaData.notificationDescription = nocAlertDescription; + notificationPayloadForNoc.metaData.mainRecipients = severeEmailRecipients; + notificationPayload.metaData.notificationId = generateNotificationId({ + notificationPrefix, + timezoneOffset: clusterInfoObj[clusterId].timezone_offset || 0, + }), + delete notificationPayloadForNoc.metaData.cc; + await NotificationQueue.create(notificationPayloadForNoc); } } + // If failed to reach cluster, do not update last monitored time in monitoring logs + const scannedClusters = clusterIds.filter( + (id) => !failedToReachClusters.includes(id) + ); + // Update monitoring logs - for (let id of clusterIds) { + for (let id of scannedClusters) { try { //Get existing metadata const log = await MonitoringLogs.findOne({ @@ -456,16 +473,11 @@ const Integrations = models.integrations; ); } } + } catch (err) { logger.error(err); } finally { - logger.debug(`Job monitoring completed started ${now} and ended at ${new Date()}`); if (parentPort) parentPort.postMessage("done"); else process.exit(0); } -})(); - - - -// TODO - Start from Intermediate -// TODO - All monitorings - update last run details +})(); \ No newline at end of file diff --git a/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js b/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js index 68df5331..8976fb99 100644 --- a/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js +++ b/Tombolo/server/jobs/jobMonitoring/monitorJobsUtil.js @@ -136,6 +136,7 @@ function setTimeToDate(date, time) { //Calculate start and end time given local time at cluster , run window, expected start and completion time function calculateStartAndEndDateTime({localDateTimeAtCluster, runWindow, expectedStartTime, expectedCompletionTime}) { let startAndEnd; + const previousDay = new Date(localDateTimeAtCluster.getTime() - 86400000); if (runWindow === "overnight") { startAndEnd = { start: setTimeToDate(previousDay, expectedStartTime || "12:00"), @@ -167,11 +168,11 @@ function calculateStartAndEndDateTime({localDateTimeAtCluster, runWindow, expect } // Daily jobs -function calculateRunOrCompleteByTimeForDailyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimeForDailyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs = 0}) { const {runWindow} = schedule; const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); let window = {frequency: "daily", currentTime: localDateTimeAtCluster}; @@ -190,14 +191,14 @@ function calculateRunOrCompleteByTimeForWeeklyJobs({ expectedStartTime, expectedCompletionTime, timezone_offset, - backDateInMinutes = 0, + backDateInMs = 0, }) { const { days, runWindow } = schedule; // Find current day at the cluster given timezone offset and this system in utc const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); const adjustedLocalTimeAtCluster = new Date( - localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000 + localDateTimeAtCluster.getTime() - backDateInMs ); const day = adjustedLocalTimeAtCluster.getDay(); const runDay = days.find((d) => d === day.toString()); @@ -219,11 +220,11 @@ function calculateRunOrCompleteByTimeForWeeklyJobs({ } // Monthly jobs -function calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs = 0}) { // Determine if schedule is by date or week and weekday const { scheduleBy } = schedule[0]; const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); let window = { currentTime: localDateTimeAtCluster, @@ -299,12 +300,12 @@ function calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime } // Yearly jobs -function calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs = 0}) { const {scheduleBy} = schedule[0]; // Local date time at cluster const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); // Current month at the cluster const monthAtCluster = adjustedLocalDateTimeAtCluster.getMonth(); @@ -398,13 +399,13 @@ function calculateRunOrCompleteByTimeForCronJobs({ expectedStartTime, expectedCompletionTime, timezone_offset, - backDateInMinutes = 0, + backDateInMs = 0, }) { const cron = schedule[0].cron; // Local date time at cluster const localDateTimeAtCluster = findLocalDateTimeAtCluster(timezone_offset); - const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMinutes * 60 * 1000); + const adjustedLocalDateTimeAtCluster = new Date(localDateTimeAtCluster.getTime() - backDateInMs); // Get the previous and next dates the cron job was supposed to run const interval = cronParser.parseExpression(cron, { @@ -463,21 +464,21 @@ function checkIfCurrentTimeIsWithinRunWindow({start, end, currentTime}) { } // Calculate run and complete by time for a job on cluster's local time -function calculateRunOrCompleteByTimes({schedule, timezone_offset, expectedStartTime, expectedCompletionTime, backDateInMinutes = 0}) { +function calculateRunOrCompleteByTimes({schedule, timezone_offset, expectedStartTime, expectedCompletionTime, backDateInMs = 0}) { // determine frequency frequency = schedule[0].frequency; switch (frequency) { case "daily": - return calculateRunOrCompleteByTimeForDailyJobs({schedule: schedule[0],expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForDailyJobs({schedule: schedule[0],expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "weekly": - return calculateRunOrCompleteByTimeForWeeklyJobs({schedule: schedule[0], expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForWeeklyJobs({schedule: schedule[0], expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "monthly": - return calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForMonthlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "yearly": - return calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForYearlyJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); case "cron": - return calculateRunOrCompleteByTimeForCronJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMinutes}); + return calculateRunOrCompleteByTimeForCronJobs({schedule, expectedStartTime, expectedCompletionTime, timezone_offset, backDateInMs}); default: throw new Error(`Unknown frequency: ${frequency}`); } @@ -603,7 +604,7 @@ function getDateReplacements(date) { } // Generate job name from job pattern -function generateJobName({ pattern, timezone_offset = 0 }) { +function generateJobName({ pattern, timezone_offset = 0, backDateInDays = 0 }) { let patternCopy = pattern; if (!patternCopy) return ""; @@ -616,7 +617,9 @@ function generateJobName({ pattern, timezone_offset = 0 }) { // No pattern, no adjustments if (dateSubstring === "") { - const date = new Date(Date.now() + timezone_offset * 1000); + let date = new Date(Date.now() + timezone_offset * 1000); + // adjust backdate + date = new Date(date.getTime() - backDateInDays * 86400000); const replacements = getDateReplacements(date); const translatedDate = replacements["%Y"] + replacements["%m"] + replacements["%d"]; @@ -636,9 +639,7 @@ function generateJobName({ pattern, timezone_offset = 0 }) { patternCopy = patternCopy.replace("", ""); - const date = new Date( - Date.now() + timezone_offset * 60000 + adjustment * 86400000 - ); + const date = new Date(Date.now() + timezone_offset * 60000 + adjustment * 86400000); const replacements = getDateReplacements(date); for (const key in replacements) { @@ -657,6 +658,29 @@ function generateJobName({ pattern, timezone_offset = 0 }) { // console.log( generateJobName({pattern: "* Test",timezone_offset: 0})); // console.log(generateJobName({ pattern: "Launch "})); +const nocAlertDescription = `[SEV TICKET REQUEST] + The following issue has been identified via automation. + Please open a sev ticket if this issue is not yet in the process of being addressed. Bridgeline not currently required.`; + + + +// Given 2 times in HH:MM format, calculate the difference in milliseconds +function differenceInMs({ startTime, endTime, daysDifference }) { + const [hours1, minutes1] = startTime.split(":").map(Number); + const [hours2, minutes2] = endTime.split(":").map(Number); + + const startDate = new Date(); + const endDate = new Date(); + + startDate.setHours(hours1, minutes1, 0, 0); + endDate.setHours(hours2, minutes2, 0, 0); + + startDate.setDate(startDate.getDate() - daysDifference); + + const difference = endDate - startDate; // Convert milliseconds to minutes + return Math.abs(difference); +} + module.exports = { matchJobName, findStartAndEndTimes, @@ -671,4 +695,6 @@ module.exports = { intermediateStates, getProductCategory, getDomain, + nocAlertDescription, + differenceInMs, }; diff --git a/Tombolo/server/jobs/notifications/processEmailNotifications.js b/Tombolo/server/jobs/notifications/processEmailNotifications.js index 9dd89d2c..9e131874 100644 --- a/Tombolo/server/jobs/notifications/processEmailNotifications.js +++ b/Tombolo/server/jobs/notifications/processEmailNotifications.js @@ -96,7 +96,7 @@ const SentNotification = models.sent_notifications; }; // Send email - await sendEmail({ ...emailPayload }); + const emailResponse = await sendEmail({ ...emailPayload }); // Assume success - if no error is thrown successfulDelivery.push(emailPayload);