Name

sn_app_insights.MetricDataCollector

Description

Logic to collect metrics data, call required classes to parse xml and update into metricbase.

Script

var MetricDataCollector = Class.create();
MetricDataCollector.prototype = {

  initialize: function() {
      this.startTime = new GlideDateTime();
  },

  persistMLMetrics: function() {
      var interval = "one";
      var metricNamesAndAggregate = {
          "semaphores": "mean",
          "sys_load": "mean"
      };

      var nodeRecord = new GlideRecord("sys_cluster_state");
      nodeRecord.query();
      while (nodeRecord.next()) {
          var parseXmlStats = new ParseXmlStats(nodeRecord.node_stats.stats);

          for (var metricName in metricNamesAndAggregate) {
              var metricValue = parseXmlStats.parseServletMetrics(metricName, interval, metricNamesAndAggregate[metricName]);
              this._saveToMetricBase(nodeRecord, metricName, metricValue);
          }
      }
  },

  persistNodeMetrics: function(interval) {
      var metricNamesAndAggregate = {
          "threads_db": "mean",
          "sql_inserts": "mean",
          "sql_response": "mean",
          "sql_selects": "mean",
          "sql_deletes": "mean",
          "replication": "mean",
          "transactions": "mean",
          "events_processed": "mean",
          "event_logs": "mean"
      };

      var resultMap = this.populatePreviousHoursMetricData();
      var nodeRecord = new GlideRecord("sys_cluster_state");
      nodeRecord.query();
      while (nodeRecord.next()) {
          var xmlStatsParser = new ParseXmlStats(nodeRecord.node_stats.stats);
          var nodeId = nodeRecord.getValue("node_id");

          for (var metricName in metricNamesAndAggregate) {
              var metricValue = xmlStatsParser.parseServletMetrics(metricName, interval, metricNamesAndAggregate[metricName]);
              this._saveToMetricBase(nodeRecord, metricName, metricValue);
          }

          // sql_response logs all database transactions, so the 'count' metric gives us the total database throughput from this node
          var databaseThroughputValue = xmlStatsParser.parseServletMetrics("sql_response", interval, "count");
          this._saveToMetricBase(nodeRecord, 'database_throughput', databaseThroughputValue);

          this._persistSemaphoreMetrics(xmlStatsParser, nodeRecord, resultMap);

          var currentTransactionCount = xmlStatsParser.parseTransactionCount();
          var rawTransactionCountMetricName = "transaction_count_raw";
          this._saveToMetricBase(nodeRecord, rawTransactionCountMetricName, currentTransactionCount);
          var transactionCountDelta = this._getRawMetricChange(nodeRecord, resultMap, rawTransactionCountMetricName, currentTransactionCount);
          this._saveToMetricBase(nodeRecord, "transaction_count", transactionCountDelta);

          var loggedInUserCount = xmlStatsParser.parseLoggedInUserCount();
          this._saveToMetricBase(nodeRecord, "logged_in_users", loggedInUserCount);
      }
  },

  populatePreviousHoursMetricData: function() {
      // Raw metrics are metrics that are stored in 2 parts:
      // * the raw value that only goes up
      // * the desired metric that is the difference between consecutive polls of the raw value
      var rawMetrics = {
          "transaction_count_raw": "",
          "default_rejections_raw": "",
          "api_int_rejections_raw": "",
          "amb_send_rejections_raw": "",
          "amb_receive_rejections_raw": "",
      };

      var resultMap = {};
      for (metric in rawMetrics) {
          var nodeRecord = new GlideRecord('sys_cluster_state');
          nodeRecord.query();

          /* Get the last hour of raw counts (in case there are recent null values)
           * This supplemental metric is needed as the value always increases and we want to know the difference
           * Other ways of calculating the difference (e.g. calculate on retrieval, using a table) are less desirable
           */
          var transformer = new sn_clotho.Transformer(nodeRecord);
          transformer.metric(metric).resample("MAX", new GlideDuration(5 * 60 * 1000));

          var start = new GlideDateTime();
          //1 hour ago
          start.addSeconds(-3600);

          var resultArray = transformer.execute(start, new GlideDateTime()).toArray();

          //Place metrics in a map for easy retrieval in the loop below
          for (var i = 0; i < resultArray.length; i++) {
              var result = resultArray[i];
              if (!resultMap[metric])
                  resultMap[metric] = {};

              resultMap[metric][result.getSubject()] = result.getValues();
          }
      }
      return resultMap;
  },

  _persistSemaphoreMetrics: function(xmlStatsParser, nodeRecord, resultMap) {
      var semaphoreMetrics = [{
              "name": "AMB_RECEIVE",
              "metrics": [{
                      "metricBaseName": "amb_receive_queue_depth",
                      "xmlStatsName": "queue_depth"
                  },
                  {
                      "metricBaseName": "amb_receive_rejections",
                      "xmlStatsName": "rejected_executions",
                      "metricBaseRawName": "amb_receive_rejections_raw"
                  }
              ]
          },
          {
              "name": "AMB_SEND",
              "metrics": [{
                      "metricBaseName": "amb_send_queue_depth",
                      "xmlStatsName": "queue_depth"
                  },
                  {
                      "metricBaseName": "amb_send_rejections",
                      "xmlStatsName": "rejected_executions",
                      "metricBaseRawName": "amb_send_rejections_raw"
                  }
              ]
          },
          {
              "name": "API_INT",
              "metrics": [{
                      "metricBaseName": "api_int_queue_depth",
                      "xmlStatsName": "queue_depth"
                  },
                  {
                      "metricBaseName": "api_int_rejections",
                      "xmlStatsName": "rejected_executions",
                      "metricBaseRawName": "api_int_rejections_raw"
                  }
              ]
          },
          {
              "name": "Default",
              "metrics": [{
                      "metricBaseName": "default_queue_depth",
                      "xmlStatsName": "queue_depth"
                  },
                  {
                      "metricBaseName": "default_rejections",
                      "xmlStatsName": "rejected_executions",
                      "metricBaseRawName": "default_rejections_raw"
                  }
              ]
          }
      ];

      semaphoreMetrics.forEach(function(semaphore) {
          var metrics = semaphore.metrics;
          metrics.forEach(function(metric) {
              var metricValue = xmlStatsParser.parseSemaphoreStatsByMetric(semaphore.name, metric.xmlStatsName);

              if (metric.metricBaseRawName) {
                  this._saveToMetricBase(nodeRecord, metric.metricBaseRawName, metricValue);
                  metricValue = this._getRawMetricChange(nodeRecord, resultMap, metric.metricBaseRawName, metricValue);
              }
              this._saveToMetricBase(nodeRecord, metric.metricBaseName, metricValue);
          }.bind(this));
      }.bind(this));
  },

  _getRawMetricChange: function(nodeRecord, resultMap, rawMetricName, rawValue) {
      if (!resultMap[rawMetricName])
          return;

      var nodeMetrics = resultMap[rawMetricName][nodeRecord.getUniqueValue()];

      var returnValue = 0;
      //If there is a raw metric count available, use it to calculate the difference
      if (nodeMetrics) {
          var lastVal = 0;
          for (var j = nodeMetrics.length - 1; j >= 0; --j) {
              var nodeVal = nodeMetrics[j];
              if (!isNaN(nodeVal)) {
                  lastVal = nodeVal;
                  break;
              }
          }

          var delta = rawValue - lastVal;

          //If delta is negative, must be new count, therefore the current value is the delta
          returnValue = delta >= 0 ? delta : rawValue;
      }
      return returnValue;
  },

  persistSlowPatternMetrics: function(slowPatternRecord) {
      var metricNamesAndValues = {
          "execution_count": "",
          "average_execution_time": "",
          "total_execution_time": ""
      };

      metricNamesAndValues["execution_count"] = slowPatternRecord.count;
      metricNamesAndValues["average_execution_time"] = slowPatternRecord.average;
      var gdt = new GlideDateTime(slowPatternRecord.total);
      metricNamesAndValues["total_execution_time"] = gdt.getNumericValue();

      for (metricName in metricNamesAndValues) {
          this._saveToMetricBase(slowPatternRecord, metricName, metricNamesAndValues[metricName]);
      }
  },

  persistEccQueueMetrics: function(queueGr) {
      if (new GlidePluginManager().isActive('com.glideapp.agent')) {
          var metricNamesAndValues = {
              "current_count_processed_input_metric": "",
              "current_count_processed_output_metric": "",
              "current_count_processing_input_metric": "",
              "current_count_processing_output_metric": "",
              "current_count_ready_input_metric": "",
              "current_count_ready_output_metric": "",
              "interval_count_processed_input_metric": "",
              "interval_count_processed_output_metric": "",
              "interval_count_processing_input_metric": "",
              "interval_count_processing_output_metric": "",
              "interval_count_ready_input_metric": "",
              "interval_count_ready_output_metric": ""
          };

          for (var metricName in metricNamesAndValues) {
              //To find the actual value to save to the metric, need to remove common suffix '_metric'
              //The suffix was added because MetricBase requires that metric names be unique, even against field names on the target table
              this._saveToMetricBase(queueGr, metricName, queueGr.getValue(metricName.replace('_metric', '')));
          }
      }
  },

  persistEventQueueMetrics: function(queueGr) {
      var gr = new GlideRecord("sys_metric");
      gr.addQuery("name", "sn_app_insights_event_queue_cluster_stats");
      gr.query();

      var metricNames = [];
      while (gr.next()) {
          metricNames.push(gr.getValue("element"));
      }

      for (var i = 0; i < metricNames.length; i++){
          metricName = metricNames[i];
          //To find the actual value to save to the metric, need to remove common suffix '_metric'
          //The suffix was added because MetricBase requires that metric names be unique, even against field names on the target table
          this._saveToMetricBase(queueGr, metricName, queueGr.getValue(metricName.replace('_metric', '')));
      }
  },

  persistScheduledJobMetrics: function(durationInMinutes) {
      var metricNames = {
          "error_count": "SUM",
          "processing_duration": "AVG",
          "run_count": "SUM"
      };

      var triggerGr = new GlideAggregate("sys_trigger");
      var sysautoGr = new GlideRecord('sysauto');

      var then = new GlideDateTime();
      then.addSeconds(-1 * durationInMinutes * 60);

      triggerGr.addNotNullQuery("document_key");
      triggerGr.addNotNullQuery("document");
      triggerGr.addQuery('sys_updated_on', '>', then);
      var condition = triggerGr.addNullQuery('system_id');
      condition.addOrCondition('system_id', 'NOT IN', ['ACTIVE NODES', 'ALL NODES']);
      for (var metricName in metricNames)
          triggerGr.addAggregate(metricNames[metricName], metricName);

      triggerGr.groupBy("document_key");

      triggerGr.query();

      while (triggerGr.next()) {
          if (!sysautoGr.get(triggerGr.getValue("document_key")))
              continue;

          for (metricName in metricNames) {
              var value = triggerGr.getAggregate(metricNames[metricName], metricName);
              if (!value)
                  continue;

              this._saveToMetricBase(sysautoGr, metricName, value);
          }
      }
  },

  _saveToMetricBase: function(record, metricName, metricValue) {
      if (gs.nil(metricName) || gs.nil(metricValue))
          return;

      var dataBuilder = new sn_clotho.DataBuilder(record, metricName).add(this.startTime, metricValue);
      new sn_clotho.Client().put(dataBuilder);
  },

  type: 'MetricDataCollector'
};

Sys ID

fd7548ee537a50109d9eddeeff7b1253

Offical Documentation

Official Docs: