Name

sn_em_ai.EvtMgmtAlertCrawler

Description

No description available

Script

var EvtMgmtAlertCrawler = Class.create();
EvtMgmtAlertCrawler.prototype = {
  initialize: function() {
      this.alertCrawlerUtils = new sn_em_ai.EvtMgmtAlertCrawlerUtils();
      this.evtMgmtCommons = new global.EvtMgmtCommons();
      this.alertDelayProcessingTime = parseInt(gs.getProperty("sn_em_ai.alert_delay_processing_time", 5), 10) || 5;
      this.queuedAlertDelayTime = parseInt(gs.getProperty("sn_em_ai.queued_alert_delay_time", 2), 10) || 2;

  },

  // SCALE OUT OPTIONS: not doing it now since its too complex
  //split between the 2 iterations; one over primary and stand aline alerts, and a seperate job for secondary alerts and changed alert tags
  // scale out the secondary alerts with multiple jobs: this can work if the job will run by parent
  process: function() {
      var slowStepManager = new global.SlowStepJSManager();
      slowStepManager.setTopic("Alert Crawler");
      slowStepManager.startStep('Start Job');

      var startTimeForPerformence = new Date().getTime();
      var totalAlertCount = 0;
      var perfCounters = {};
      perfCounters.numberOfAlert = 0;
      perfCounters.numberOfChangedAlerts = 0;
      perfCounters.durationOfAlertProcessing = 0;
      perfCounters.durationOfChangedAlertProcessing = 0;
      perfCounters.durationOfInsertingToUtilityTables = 0;
      perfCounters.maxAlertProcessingDuration = {
          value: 0,
          alert: ""
      };
      perfCounters.maxChangedAlertProcessingDuration = {
          value: 0,
          alert: "",
      };
      var startTimeForAlertProcessing = new Date().getTime();
      var duration = 0;

      slowStepManager.startStep('Get Hash');
      var alertProcessTimeForAlerts = new GlideDateTime();
      alertProcessTimeForAlerts.subtract(this.alertDelayProcessingTime * 1000);

      var hashValue = this.alertCrawlerUtils.getHashDefaultEmpty(this.alertCrawlerUtils.ALERT_CRAWLER_TRIGGER);
      var hashDateTime = new GlideDateTime(); // for the first run - we start to run on alerts created now (no history)
      if (hashValue) {
          hashDateTime.setValue(hashValue); // set with the hash value, if exists
      }

      slowStepManager.startStep('Get relavent stand alone and primary alerts');
      var alertsGr = this.alertCrawlerUtils.getAlertsBetweenTimes(alertProcessTimeForAlerts, hashDateTime);

      var limitReached = alertsGr.getRowCount() >= this.alertCrawlerUtils.alertProcessingQueryLimit;
      var lastAlertCreation = alertProcessTimeForAlerts;
      slowStepManager.startStep('Iterate over stand alone and primary alerts');

      while (alertsGr.next()) {
          this.evtMgmtCommons.addDebugLogNoPrefix(this.type + ': time range for stand alone and primary alerts retrieval is:  ' + hashDateTime + '-' + alertProcessTimeForAlerts);

          totalAlertCount++;
          perfCounters.numberOfAlert++;
          // Take time before processing an alert
          var timeBefore = new Date();

          if (limitReached) {
              lastAlertCreation = alertsGr.getValue(this.alertCrawlerUtils.SYS_CREATED_ON);
          }

          var alertId = alertsGr.getUniqueValue();
          var ciId = alertsGr.getValue(this.alertCrawlerUtils.CMDB_CI);
          var alertDomain = this.evtMgmtCommons.getRecordDomain(alertsGr);
          if (ciId) {
              if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
                  var ciName = alertsGr.getDisplayValue(this.alertCrawlerUtils.CMDB_CI);
                  var ciClass = alertsGr.cmdb_ci.sys_class_name;
                  this.alertCrawlerUtils.addToAlertCIsTable(alertId, ciId, ciName, ciClass, alertDomain);
              }
              if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
                  this.alertCrawlerUtils.handleImpactedService(alertId, ciId, alertDomain, false);
              }
          } else {
              if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
                  this.alertCrawlerUtils.addToAlertCIsTable(alertId, "", "", "", alertDomain);
              }
              if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
                  this.alertCrawlerUtils.addToAlertServicesTable(alertId, "", "", "", alertDomain, "");
              }
          }

          //handle alert tags
          if (this.alertCrawlerUtils.POPULATE_ALERT_TAGS) {
              var alertTagsField = this.alertCrawlerUtils.parseJsonReturnEmptyDefault(alertsGr.getValue(this.alertCrawlerUtils.ALERT_TAGS));
              // Remove all records for alert, to avoid duplicate records
              this.alertCrawlerUtils.removeAlertTagsForAlert(alertId);
              for (var alertTag in alertTagsField) {
                  this.alertCrawlerUtils.addToAlertTagTable(alertId, alertTag, alertTagsField[alertTag], alertDomain);
              }
          }

          // Take time after processing the alert
          var timeAfter = new Date();

          // Save the max time of processed alert and its related alert number
          if (perfCounters && ((timeAfter - timeBefore) >= perfCounters.maxAlertProcessingDuration.value)) {
              perfCounters.maxAlertProcessingDuration.value = timeAfter - timeBefore;
              perfCounters.maxAlertProcessingDuration.alert = alertsGr.getValue(this.alertCrawlerUtils.NUMBER);
          }
      }


      if (limitReached) {
          lastAlertCreation = new GlideDateTime(lastAlertCreation);
          var diffInSeconds = (lastAlertCreation.getNumericValue() / 1000 | 0) - (hashDateTime.getNumericValue() / 1000 | 0);
          if (diffInSeconds == 0) {
              // abort processing more alerts from this second and move on to the next second.
              gs.error(this.type + ": The number of alerts has reached the limit of " + this.alertCrawlerUtils.alertProcessingQueryLimit + " alerts, all were created in the same second: " + lastAlertCreation + ".\n" +
                  "Some alerts may not have been processed by the 'Event Management - Alert Crawler' job.\n");
              lastAlertCreation.addSeconds(1);
          }
      }

      this.evtMgmtCommons.setHashGr(this.alertCrawlerUtils.ALERT_CRAWLER_TRIGGER, lastAlertCreation);

      // save the duration of iteration over stand alone and primary alerts
      duration = (new Date().getTime()) - startTimeForAlertProcessing;
      perfCounters.durationOfAlertProcessing = duration;

      //handle changed alerts: secondaries that were added/removed from groups, and alert with changed alert tags
      var startTimeForChangedAlertProcessing = new Date().getTime();
      slowStepManager.startStep('Get changed aggregated alerts');

      var queuedAlertDelayTime = new GlideDateTime();
      queuedAlertDelayTime.subtract(this.queuedAlertDelayTime * 1000);
      var changedSecondariesGr = this.alertCrawlerUtils.getChangedAlerts(queuedAlertDelayTime);

      slowStepManager.startStep('Iterate over aggregated alerts');
      while (changedSecondariesGr.next()) {
          totalAlertCount++;
          perfCounters.numberOfChangedAlerts++;
          try {
              // Take time before processing an alert
              timeBefore = new Date();

              //update state to "processing"
              changedSecondariesGr.state = this.alertCrawlerUtils.STATES.PROCESSING;
              changedSecondariesGr.update();

              var currentAlert = changedSecondariesGr.getValue(this.alertCrawlerUtils.ALERT);
              var alertDomain = this.evtMgmtCommons.getRecordDomain(changedSecondariesGr);
              var actionType = changedSecondariesGr.getValue(this.alertCrawlerUtils.ACTION);
              var additionalInfo = this.alertCrawlerUtils.parseJsonReturnEmptyDefault(changedSecondariesGr.getValue(this.alertCrawlerUtils.ADDITIONAL_INFO));
              switch (actionType) {
                  case this.alertCrawlerUtils.ACTIONS.ALERT_ADDED_TO_GROUP:

                      var alertCi = changedSecondariesGr.getValue(this.alertCrawlerUtils.CMDB_CI);
                      var currParent = changedSecondariesGr.getValue(this.alertCrawlerUtils.PARENT);

                      // remove stand alone records for alert- if the alert didn't have a parent before
                      this.alertCrawlerUtils.removeAllAlertRecords(currentAlert);

                      if (alertCi) { // No need to process secondary alerts without CIs
                          if (additionalInfo.hasOwnProperty(this.alertCrawlerUtils.IS_LA_ALERT) && additionalInfo[this.alertCrawlerUtils.IS_LA_ALERT]) {
                              var grandParent = changedSecondariesGr.parent.parent;
                              currParent = grandParent ? grandParent : currParent;
                          }
                          if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
                              var ciName = changedSecondariesGr.getDisplayValue(this.alertCrawlerUtils.CMDB_CI);
                              var ciClass = changedSecondariesGr.cmdb_ci.sys_class_name;
                              this.alertCrawlerUtils.addToAlertCIsTable(currParent, alertCi, ciName, ciClass, alertDomain);
                          }
                          if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
                              this.alertCrawlerUtils.handleImpactedService(currParent, alertCi, alertDomain, true);
                          }
                      }
                      break;
                  case this.alertCrawlerUtils.ACTIONS.ALERT_REMOVED_FROM_GROUP:
                      //TODO: add implementation for removing alerts from group
                      break;
                  case this.alertCrawlerUtils.ACTIONS.ALERT_TAGS_CHANGED:

                      if (this.alertCrawlerUtils.POPULATE_ALERT_TAGS) {
                          this.alertCrawlerUtils.removeAlertTagsForAlert(currentAlert);
                          var alertTagsField = this.alertCrawlerUtils.parseJsonReturnEmptyDefault(changedSecondariesGr.getValue(this.alertCrawlerUtils.ALERT_TAGS));
                          for (var alertTag in alertTagsField) {
                              this.alertCrawlerUtils.addToAlertTagTable(currentAlert, alertTag, alertTagsField[alertTag], alertDomain);
                          }
                      }
                      break;

              }

              //set status to "complete"
              changedSecondariesGr.state = this.alertCrawlerUtils.STATES.COMPLETE;
              changedSecondariesGr.update();

              // Take time after processing the alert
              timeAfter = new Date();

              // Save the max time of processed alert and its related alert number
              if (perfCounters && ((timeAfter - timeBefore) >= perfCounters.maxChangedAlertProcessingDuration.value)) {
                  perfCounters.maxChangedAlertProcessingDuration.value = timeAfter - timeBefore;
                  perfCounters.maxChangedAlertProcessingDuration.alert = String(changedSecondariesGr.alert.number);
              }
          } catch (e) {
              gs.error(this.evtMgmtCommons.getExceptionMessage(e, true));
              changedSecondariesGr.state = this.alertCrawlerUtils.STATES.FAILED;
              changedSecondariesGr.update();
          }
      }

      // save the duration of iterating over changed alerts
      duration = (new Date().getTime()) - startTimeForChangedAlertProcessing;
      perfCounters.durationOfChangedAlertProcessing = duration;

      //insert records in batch
      timeBefore = new Date();
      slowStepManager.startStep('Insert in to utility tables in batches');
      if (this.alertCrawlerUtils.POPULATE_ALERT_SERVICES) {
          this.alertCrawlerUtils.insertAlertServicesBatchRecords();
      }
      if (this.alertCrawlerUtils.POPULATE_ALERT_CIS) {
          this.alertCrawlerUtils.insertAlertCisBatchRecords();
      }
      if (this.alertCrawlerUtils.POPULATE_ALERT_TAGS) {
          this.alertCrawlerUtils.insertAlertTagsBatchRecords();
      }
      duration = (new Date().getTime()) - timeBefore;
      perfCounters.durationOfInsertingToUtilityTables = duration;

      slowStepManager.startStep('Write to performance statistics');
      //write to performance statistics
      var perfJson = perfCounters;
      perfJson.processedAlerts = totalAlertCount;
      perfJson.startAlertCreationTime = String(hashDateTime);
      perfJson.lastAlertCreationTime = String(lastAlertCreation);

      duration = (new Date().getTime()) - startTimeForPerformence;
      this.evtMgmtCommons.writeToPerfTable("evtMgmtAlertCrawlerJob", totalAlertCount, duration, perfJson);

      slowStepManager.report();
  },

  type: 'EvtMgmtAlertCrawler'
};

Sys ID

54d0263b07f62510b34ce06b0fd300fd

Offical Documentation

Official Docs: