Name

sn_nlu_workbench.NLUPerformanceUtil

Description

Utilities to fetch NLU Performance records

Script

var NLUPerformanceUtil = Class.create();

var OUTCOME_QUERIES = {
  CORRECT: 'correct,correct_among_multiple',
  SINGLE_CORRECT: 'correct',
  CORRECT_AMONG_MULTIPLE: 'correct_among_multiple',
  INCORRECT: 'incorrect,incorrect_among_multiple,uncategorized',
  SINGLE_INCORRECT: 'incorrect',
  INCORRECT_AMONG_MULTIPLE: 'incorrect_among_multiple,uncategorized',
  UNKNOWN: 'skipped,uncategorized',
  NO_PREDICTION: 'skipped',
  SINGLE_UNKNOWN: 'uncategorized'
};

var tableName = 'open_nlu_predict_intent_feedback';

NLUPerformanceUtil.prototype = {
  initialize: function() {},

  getAuditLogByTask: function(task) {
      var gr = new GlideRecord(tableName);
      gr.addQuery('app_document', task);
      gr.chooseWindow(0, 1);
      gr.query();

      while (gr.next()) {
          return gr.getValue('audit_log');
      }
      return null;
  },

  getDynamicallyTranslatedLog: function() {
      var translatedLogs = {};

      var gr = new GlideRecord('sys_cs_message');
      gr.addEncodedQuery('va_system_payload!=NULL');
      gr.query();
      while (gr.next()) {
          var task = gr.getValue('task');
          var log = this.getAuditLogByTask(task);
          if (log) {
              translatedLogs[log] = 1;
          }
      }
      return translatedLogs;
  },

  getLogsMapByOutcome: function(selectedId, encodedQuery, outcomes, ignoreLogs) {
      // CORRECT, SINGLE_CORRECT, CORRECT_AMONG_MULTIPLE, SINGLE_INCORRECT and NO_PREDICTION cases
      var auditLogMap = outcomes ? ignoreLogs : {};

      var gr = new GlideRecord(tableName);
      var outcomeQuery = encodedQuery + '^nlu_discovery_outcomeIN' + (outcomes || OUTCOME_QUERIES[selectedId]);
      gr.addEncodedQuery(outcomeQuery);
      gr.query();
      while (gr.next()) {
          var log = gr.getValue('audit_log');
          if (outcomes || !ignoreLogs[log]) {
              auditLogMap[log] = 1;
          }
      }
      return auditLogMap;
  },

  getLogsByAggregration: function(selectedId, encodedQuery, ignoreLogs, isMultiple) {
      // INCORRECT, INCORRECT_AMONG_MULTIPLE, UNKNOWN and SINGLE_UNKNOWN cases
      var logs = [];
      var agg = new GlideAggregate(tableName);
      var outcomeQuery = encodedQuery + '^nlu_discovery_outcomeIN' + OUTCOME_QUERIES[selectedId];
      agg.addAggregate('COUNT', 'audit_log');
      agg.addEncodedQuery(outcomeQuery);
      agg.query();
      while (agg.next()) {
          var count = parseInt(agg.getAggregate('COUNT', 'audit_log'));
          var countCheck = isMultiple ? count > 1 : count === 1;
          var log = agg.getValue('audit_log');
          if (countCheck && !ignoreLogs[log]) {
              logs.push(log);
          }
      }
      return logs;
  },

  getAuditLogs: function(selectedId, encodedQuery, translatedLogMap) {
      // TODO: Decouple the logic into single use-cases
      var skipOutcome = '',
          logMap = {};
      if (selectedId === 'UNKNOWN' || selectedId === 'SINGLE_UNKNOWN') {
          skipOutcome = OUTCOME_QUERIES.CORRECT + ',' + OUTCOME_QUERIES.SINGLE_INCORRECT + ',incorrect_among_multiple';
          logMap = this.getLogsMapByOutcome(selectedId, encodedQuery, skipOutcome, translatedLogMap);
          return this.getLogsByAggregration(selectedId, encodedQuery, logMap, false);
      } else if (selectedId === 'INCORRECT_AMONG_MULTIPLE' || selectedId === 'INCORRECT') {
          skipOutcome = OUTCOME_QUERIES.CORRECT;
          if (selectedId === 'INCORRECT_AMONG_MULTIPLE') {
              skipOutcome += ',' + OUTCOME_QUERIES.SINGLE_INCORRECT;
          }
          logMap = this.getLogsMapByOutcome(selectedId, encodedQuery, skipOutcome, translatedLogMap);
          var multiplePredOutcome = this.getLogsByAggregration(selectedId, encodedQuery, logMap, true);
          if (selectedId !== 'INCORRECT_AMONG_MULTIPLE') {
              var singlePredOutcome = this.getLogsByAggregration('SINGLE_INCORRECT', encodedQuery, logMap, false);
              for (var i = 0; i < singlePredOutcome.length; ++i) {
                  if (multiplePredOutcome.indexOf(singlePredOutcome[i]) === -1)
                      multiplePredOutcome.push(singlePredOutcome[i]);
              }
          }
          return multiplePredOutcome;
      }
      return Object.keys(this.getLogsMapByOutcome(selectedId, encodedQuery, null, translatedLogMap));
  },

  getPerformanceInfoCount: function(encodedQuery) {
      var gr = new GlideAggregate(tableName);
      gr.addEncodedQuery(encodedQuery);
      gr.addAggregate('COUNT');
      gr.query();
      return gr.next() ? parseInt(gr.getAggregate('COUNT')) : 0;
  },

  getPerformanceInfo: function(selectedIds, encodedQuery, translatedLogMap, offset, limit) {
      if (selectedIds.length === 0) {
          return {
              data: [],
              count: 0
          };
      }
      var auditLogs = [],
          performanceQuery = encodedQuery;
      if (selectedIds.indexOf('ALL_OUTCOME') < 0) {
          var _self = this;
          selectedIds.forEach(function(selectedId) {
              auditLogs = auditLogs.concat(_self.getAuditLogs(selectedId, encodedQuery, translatedLogMap));
          });

          if (auditLogs.length > 0) {
              performanceQuery += '^audit_logIN' + auditLogs.join(',');
          }
      } else {
          performanceQuery += '^audit_logNOT IN' + Object.keys(translatedLogMap).join(',');
      }

      var count = this.getPerformanceInfoCount(performanceQuery);
      performanceQuery += '^ORDERBYutterance,sys_created_on,audit_log,nlu_discovery_outcome';
      var gr = new GlideRecord(tableName);
      gr.addEncodedQuery(performanceQuery);
      gr.chooseWindow(offset > 0 ? offset - 1 : 0, offset + limit + 1); // -1 and +1 is to cover =1 and >1 uncategorised outcome distribution
      gr.query();
      var id = '',
          predictionInfo = {},
          i = 0,
          data = [];
      while (gr.next()) {
          if (id === gr.getValue('audit_log') && (offset === 0 || i != 1)) {
              predictionInfo = {
                  id: gr.getValue('audit_log') + "-" + Math.random(),
                  createdOn: gr.getDisplayValue('sys_created_on'),
                  modelName: gr.getDisplayValue('nlu_model'),
                  prediction: gr.getValue('prediction'),
                  utterance: gr.getValue('utterance')
              };
              if (gr.getValue('nlu_discovery_outcome') === data[i - 1].outcome) {
                  // this overriding is needed for multiple uncategorised outcome which should be grouped as incorrect among multiple
                  data[i - 1].outcome = 'incorrect_among_multiple';
              }
          } else {
              id = gr.getValue('audit_log');
              predictionInfo = {
                  id: gr.getValue('audit_log'),
                  createdOn: gr.getDisplayValue('sys_created_on'),
                  groupedUtterance: gr.getValue('utterance'),
                  modelName: gr.getDisplayValue('nlu_model'),
                  outcome: gr.getValue('nlu_discovery_outcome'),
                  prediction: gr.getValue('prediction'),
                  utterance: gr.getValue('utterance')
              };
              if (offset !== 0 && i === 1 && gr.getValue('nlu_discovery_outcome') === data[0].outcome) {
                  // this overriding is needed for multiple uncategorised outcome which should be grouped as incorrect among multiple
                  predictionInfo.outcome = 'incorrect_among_multiple';
              }
          }
          data.push(predictionInfo);
          ++i;
      }
      return {
          data: data.splice(offset > 0 ? 1 : 0, limit),
          count: count
      };
  },

  type: 'NLUPerformanceUtil'
};

Sys ID

841780ed77ef0110160a0adc3c5a9971

Offical Documentation

Official Docs: