Name
sn_em_ai.EvtMgmtAlertCrawler
Description
No description available
Script
var EvtMgmtAlertCrawler = Class.create();
EvtMgmtAlertCrawler.prototype = {
initialize: function() {
this.alertCrawlerUtils = new sn_em_ai.EvtMgmtAlertCrawlerUtils();
this.evtMgmtCommons = new global.EvtMgmtCommons();
this.alertDelayProcessingTime = parseInt(gs.getProperty("sn_em_ai.alert_delay_processing_time", 5), 10) || 5;
this.queuedAlertDelayTime = parseInt(gs.getProperty("sn_em_ai.queued_alert_delay_time", 2), 10) || 2;
},
// SCALE OUT OPTIONS: not doing it now since its too complex
//split between the 2 iterations; one over primary and stand aline alerts, and a seperate job for secondary alerts and changed alert tags
// scale out the secondary alerts with multiple jobs: this can work if the job will run by parent
process: function() {
var slowStepManager = new global.SlowStepJSManager();
slowStepManager.setTopic("Alert Crawler");
slowStepManager.startStep('Start Job');
var startTimeForPerformence = new Date().getTime();
var totalAlertCount = 0;
var perfCounters = {};
perfCounters.numberOfAlert = 0;
perfCounters.numberOfChangedAlerts = 0;
perfCounters.durationOfAlertProcessing = 0;
perfCounters.durationOfChangedAlertProcessing = 0;
perfCounters.durationOfInsertingToUtilityTables = 0;
perfCounters.maxAlertProcessingDuration = {
value: 0,
alert: ""
};
perfCounters.maxChangedAlertProcessingDuration = {
value: 0,
alert: "",
};
var startTimeForAlertProcessing = new Date().getTime();
var duration = 0;
slowStepManager.startStep('Get Hash');
var alertProcessTimeForAlerts = new GlideDateTime();
alertProcessTimeForAlerts.subtract(this.alertDelayProcessingTime * 1000);
var hashValue = this.alertCrawlerUtils.getHashDefaultEmpty(this.alertCrawlerUtils.ALERT_CRAWLER_TRIGGER);
var hashDateTime = new GlideDateTime(); // for the first run - we start to run on alerts created now (no history)
if (hashValue) {
hashDateTime.setValue(hashValue); // set with the hash value, if exists
}
slowStepManager.startStep('Get relavent stand alone and primary alerts');
var alertsGr = this.alertCrawlerUtils.getAlertsBetweenTimes(alertProcessTimeForAlerts, hashDateTime);
var limitReached = alertsGr.getRowCount() >= this.alertCrawlerUtils.alertProcessingQueryLimit;
var lastAlertCreation = alertProcessTimeForAlerts;
slowStepManager.startStep('Iterate over stand alone and primary alerts');
while (alertsGr.next()) {
this.evtMgmtCommons.addDebugLogNoPrefix(this.type + ': time range for stand alone and primary alerts retrieval is: ' + hashDateTime + '-' + alertProcessTimeForAlerts);
totalAlertCount++;
perfCounters.numberOfAlert++;
// Take time before processing an alert
var timeBefore = new Date();
if (limitReached) {
lastAlertCreation = alertsGr.getValue(this.alertCrawlerUtils.SYS_CREATED_ON);
}
var alertId = alertsGr.getUniqueValue();
var ciId = alertsGr.getValue(this.alertCrawlerUtils.CMDB_CI);
var alertDomain = this.evtMgmtCommons.getRecordDomain(alertsGr);
if (ciId) {
if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
var ciName = alertsGr.getDisplayValue(this.alertCrawlerUtils.CMDB_CI);
var ciClass = alertsGr.cmdb_ci.sys_class_name;
this.alertCrawlerUtils.addToAlertCIsTable(alertId, ciId, ciName, ciClass, alertDomain);
}
if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
this.alertCrawlerUtils.handleImpactedService(alertId, ciId, alertDomain, false);
}
} else {
if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
this.alertCrawlerUtils.addToAlertCIsTable(alertId, "", "", "", alertDomain);
}
if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
this.alertCrawlerUtils.addToAlertServicesTable(alertId, "", "", "", alertDomain, "");
}
}
//handle alert tags
if (this.alertCrawlerUtils.POPULATE_ALERT_TAGS) {
var alertTagsField = this.alertCrawlerUtils.parseJsonReturnEmptyDefault(alertsGr.getValue(this.alertCrawlerUtils.ALERT_TAGS));
// Remove all records for alert, to avoid duplicate records
this.alertCrawlerUtils.removeAlertTagsForAlert(alertId);
for (var alertTag in alertTagsField) {
this.alertCrawlerUtils.addToAlertTagTable(alertId, alertTag, alertTagsField[alertTag], alertDomain);
}
}
// Take time after processing the alert
var timeAfter = new Date();
// Save the max time of processed alert and its related alert number
if (perfCounters && ((timeAfter - timeBefore) >= perfCounters.maxAlertProcessingDuration.value)) {
perfCounters.maxAlertProcessingDuration.value = timeAfter - timeBefore;
perfCounters.maxAlertProcessingDuration.alert = alertsGr.getValue(this.alertCrawlerUtils.NUMBER);
}
}
if (limitReached) {
lastAlertCreation = new GlideDateTime(lastAlertCreation);
var diffInSeconds = (lastAlertCreation.getNumericValue() / 1000 | 0) - (hashDateTime.getNumericValue() / 1000 | 0);
if (diffInSeconds == 0) {
// abort processing more alerts from this second and move on to the next second.
gs.error(this.type + ": The number of alerts has reached the limit of " + this.alertCrawlerUtils.alertProcessingQueryLimit + " alerts, all were created in the same second: " + lastAlertCreation + ".\n" +
"Some alerts may not have been processed by the 'Event Management - Alert Crawler' job.\n");
lastAlertCreation.addSeconds(1);
}
}
this.evtMgmtCommons.setHashGr(this.alertCrawlerUtils.ALERT_CRAWLER_TRIGGER, lastAlertCreation);
// save the duration of iteration over stand alone and primary alerts
duration = (new Date().getTime()) - startTimeForAlertProcessing;
perfCounters.durationOfAlertProcessing = duration;
//handle changed alerts: secondaries that were added/removed from groups, and alert with changed alert tags
var startTimeForChangedAlertProcessing = new Date().getTime();
slowStepManager.startStep('Get changed aggregated alerts');
var queuedAlertDelayTime = new GlideDateTime();
queuedAlertDelayTime.subtract(this.queuedAlertDelayTime * 1000);
var changedSecondariesGr = this.alertCrawlerUtils.getChangedAlerts(queuedAlertDelayTime);
slowStepManager.startStep('Iterate over aggregated alerts');
while (changedSecondariesGr.next()) {
totalAlertCount++;
perfCounters.numberOfChangedAlerts++;
try {
// Take time before processing an alert
timeBefore = new Date();
//update state to "processing"
changedSecondariesGr.state = this.alertCrawlerUtils.STATES.PROCESSING;
changedSecondariesGr.update();
var currentAlert = changedSecondariesGr.getValue(this.alertCrawlerUtils.ALERT);
var alertDomain = this.evtMgmtCommons.getRecordDomain(changedSecondariesGr);
var actionType = changedSecondariesGr.getValue(this.alertCrawlerUtils.ACTION);
var additionalInfo = this.alertCrawlerUtils.parseJsonReturnEmptyDefault(changedSecondariesGr.getValue(this.alertCrawlerUtils.ADDITIONAL_INFO));
switch (actionType) {
case this.alertCrawlerUtils.ACTIONS.ALERT_ADDED_TO_GROUP:
var alertCi = changedSecondariesGr.getValue(this.alertCrawlerUtils.CMDB_CI);
var currParent = changedSecondariesGr.getValue(this.alertCrawlerUtils.PARENT);
// remove stand alone records for alert- if the alert didn't have a parent before
this.alertCrawlerUtils.removeAllAlertRecords(currentAlert);
if (alertCi) { // No need to process secondary alerts without CIs
if (additionalInfo.hasOwnProperty(this.alertCrawlerUtils.IS_LA_ALERT) && additionalInfo[this.alertCrawlerUtils.IS_LA_ALERT]) {
var grandParent = changedSecondariesGr.parent.parent;
currParent = grandParent ? grandParent : currParent;
}
if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
var ciName = changedSecondariesGr.getDisplayValue(this.alertCrawlerUtils.CMDB_CI);
var ciClass = changedSecondariesGr.cmdb_ci.sys_class_name;
this.alertCrawlerUtils.addToAlertCIsTable(currParent, alertCi, ciName, ciClass, alertDomain);
}
if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
this.alertCrawlerUtils.handleImpactedService(currParent, alertCi, alertDomain, true);
}
}
break;
case this.alertCrawlerUtils.ACTIONS.ALERT_REMOVED_FROM_GROUP:
//TODO: add implementation for removing alerts from group
break;
case this.alertCrawlerUtils.ACTIONS.ALERT_TAGS_CHANGED:
if (this.alertCrawlerUtils.POPULATE_ALERT_TAGS) {
this.alertCrawlerUtils.removeAlertTagsForAlert(currentAlert);
var alertTagsField = this.alertCrawlerUtils.parseJsonReturnEmptyDefault(changedSecondariesGr.getValue(this.alertCrawlerUtils.ALERT_TAGS));
for (var alertTag in alertTagsField) {
this.alertCrawlerUtils.addToAlertTagTable(currentAlert, alertTag, alertTagsField[alertTag], alertDomain);
}
}
break;
}
//set status to "complete"
changedSecondariesGr.state = this.alertCrawlerUtils.STATES.COMPLETE;
changedSecondariesGr.update();
// Take time after processing the alert
timeAfter = new Date();
// Save the max time of processed alert and its related alert number
if (perfCounters && ((timeAfter - timeBefore) >= perfCounters.maxChangedAlertProcessingDuration.value)) {
perfCounters.maxChangedAlertProcessingDuration.value = timeAfter - timeBefore;
perfCounters.maxChangedAlertProcessingDuration.alert = String(changedSecondariesGr.alert.number);
}
} catch (e) {
gs.error(this.evtMgmtCommons.getExceptionMessage(e, true));
changedSecondariesGr.state = this.alertCrawlerUtils.STATES.FAILED;
changedSecondariesGr.update();
}
}
// save the duration of iterating over changed alerts
duration = (new Date().getTime()) - startTimeForChangedAlertProcessing;
perfCounters.durationOfChangedAlertProcessing = duration;
//insert records in batch
timeBefore = new Date();
slowStepManager.startStep('Insert in to utility tables in batches');
if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
this.alertCrawlerUtils.insertAlertServicesBatchRecords();
}
if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
this.alertCrawlerUtils.insertAlertCisBatchRecords();
}
if (this.alertCrawlerUtils.POPULATE_ALERT_TAGS) {
this.alertCrawlerUtils.insertAlertTagsBatchRecords();
}
duration = (new Date().getTime()) - timeBefore;
perfCounters.durationOfInsertingToUtilityTables = duration;
slowStepManager.startStep('Write to performance statistics');
//write to performance statistics
var perfJson = perfCounters;
perfJson.processedAlerts = totalAlertCount;
perfJson.startAlertCreationTime = String(hashDateTime);
perfJson.lastAlertCreationTime = String(lastAlertCreation);
duration = (new Date().getTime()) - startTimeForPerformence;
this.evtMgmtCommons.writeToPerfTable("evtMgmtAlertCrawlerJob", totalAlertCount, duration, perfJson);
slowStepManager.report();
},
type: 'EvtMgmtAlertCrawler'
};
Sys ID
54d0263b07f62510b34ce06b0fd300fd