Name

sn_nlu_workbench.NLUBatchTestDefinition

Description

Utilities to run a batch test defintion and schedule job to execute test run

Script

var NLUBatchTestDefinition = Class.create();

(function() {

  var tables = NLUWorkbenchConstants.tables;
  var constants = NLUWorkbenchConstants.constants;
  var sysProps = NLUWorkbenchConstants.sysProps;
  var EXECUTION_STATUS = NLUWorkbenchConstants.EXECUTION_STATUS;
  var FIELDS = {
      TEST_SET: 'test_set',
      MODELS: 'models',
      IS_OPTIMIZE: 'optimize'
  };

  function isOptimizeSupported(modelSnapshot) {
      var language = modelSnapshot.language || constants.DEFAULT_LANGUAGE;
      var modelVersion = modelSnapshot.version;
      var response = {
          status: 'success'
      };
      var supportInfo = NLUBatchTestIntegrator.getOptimizeSupportedLanguages();
      if (supportInfo.status === 'success') {
          var optimizeSupportMap = supportInfo.optimizeSupportMap;
          if (optimizeSupportMap[language]) {
              if (optimizeSupportMap[language].nonSupportedVersions.indexOf(modelVersion) > -1) {
                  response.status = 'failure';
                  response.message = gs.getMessage('The model needs to be retrained with the latest NLU Service version before using this feature. Please retrain before running this analysis.');
              }
          } else {
              var supportedLangLabels = Object.keys(optimizeSupportMap).map(function(lang) {
                  return optimizeSupportMap[lang].label;
              }).join(', ');
              response.status = 'failure';
              response.message = gs.getMessage("The language of the selected model isn't supported by Optimize, please choose a model whose language is/one of {0}", supportedLangLabels);
          }
      } else {
          response.status = 'failure';
          response.message = gs.getMessage('Error while fetching optimization support for model: {0} ({1})', [modelVersion, language]);
      }
      return response;
  }

  /*
  Validations: 
  	- Atleast one model
  	- If optimize, model size == 1 
  	- All the models should be:
  		- of same version
  		- of same language
  */
  NLUBatchTestDefinition.modelSnapshot = null; // This will be null for every REST call.
  NLUBatchTestDefinition.validateAndGetModelSnapshot = function(modelIds, isOptimize) {
      if (modelIds.length === 0) throw new Error(gs.getMessage('Model list is empty'));
      if (isOptimize && modelIds.length > 1) {
          throw new Error(gs.getMessage('Optimisation cannot be run on more than one model'));
      }

      var modelVersion = null,
          modelLanguage = null;
      NLUBatchTestDefinition.modelSnapshot = [];
      modelIds && modelIds.forEach(function(modelId) {
          var snapshot = NLUMLSolutionUtil.getModelSolutionInfo(modelId);
          if (isOptimize) {
              var response = isOptimizeSupported(snapshot);
              if (response.status === 'failure')
                  throw new Error(response.message);
          }

          if (!modelVersion) modelVersion = snapshot.version;
          if (!modelLanguage) modelLanguage = snapshot.language;

          if (modelVersion !== snapshot.version)
              throw new Error(gs.getMessage('Models must be of same version. Please re-train models for latest version.'));
          if (modelLanguage !== snapshot.language)
              throw new Error(gs.getMessage('Models must be of same language.'));

          NLUBatchTestDefinition.modelSnapshot.push(snapshot);
      });
      return NLUBatchTestDefinition.modelSnapshot;
  };

  NLUBatchTestDefinition.addRecord = function(testSetId, modelIds, isOptimize) {
      var record = new GlideRecord(tables.NLU_BATCH_TEST_RUN_DEFINITION);
      record.initialize();
      record.setValue(FIELDS.TEST_SET, testSetId);
      record.setValue(FIELDS.MODELS, modelIds);
      record.setValue(FIELDS.IS_OPTIMIZE, !!isOptimize);
      return record.insert();
  };

  NLUBatchTestDefinition.getRecord = function(testSetId, modelIds, isOptimize) {
      var record = new GlideRecord(tables.NLU_BATCH_TEST_RUN_DEFINITION);
      record.addQuery(FIELDS.TEST_SET, testSetId);
      record.addQuery(FIELDS.MODELS, modelIds);
      record.addQuery(FIELDS.IS_OPTIMIZE, !!isOptimize);
      record.orderByDesc('sys_updated_on');
      record.setLimit(1);
      record.query();
      return record.next() && record;
  };

  NLUBatchTestDefinition.runTestInBackground = function(definitionId, sync) {
      try {
          var testDefinition = new NLUBatchTestDefinition(definitionId);
          if (testDefinition.isExecutionInProgress())
              throw new Error(gs.getMessage('Already an execution is running for this test definition'));

          var modelSnapshot = testDefinition.getModelSnapshot();
          var testSetSnapshot = testDefinition.getTestSetSnapshot();
          var executionId = NLUBatchTestExecution.addRecord(definitionId, modelSnapshot, testSetSnapshot);
          if (!gs.nil(executionId)) testDefinition.purgeExecutions();

          if (!sync) {
              var script = "var result = new sn_nlu_workbench.NLUBatchTestExecution('" + executionId + "').start();";
              script += "gs.info('NLUBatchTestExecution.start result: ' + JSON.stringify(result));";
              global.NLUWorkbenchGlobalScript.scheduleScript(script, tables.NLU_BATCH_TEST_RUN_EXECUTION, executionId);
              return {
                  status: 'success',
                  definitionId: definitionId,
                  executionId: executionId
              };
          } else {
              return new sn_nlu_workbench.NLUBatchTestExecution(executionId).start();
          }
      } catch (e) {
          return {
              status: 'failure',
              message: e.message
          };
      }
  };

  NLUBatchTestDefinition.prototype = {

      initialize: function(definitionId) {
          this.definitionId = definitionId;
      },

      getGR: function() {
          if (!gs.nil(this.gr)) return this.gr;
          this.gr = new GlideRecord(tables.NLU_BATCH_TEST_RUN_DEFINITION);
          if (this.gr.get(this.definitionId))
              return this.gr;
          else
              throw new Error(gs.getMessage('Invalid batch test definition'));
      },

      isExecutionInProgress: function() {
          this.getGR();

          // Check if there is already a test run is in progress:
          var inProgressStatuses = EXECUTION_STATUS.PREPARING + EXECUTION_STATUS.INPROGRESS;
          var executionGr = this.getExecutionGr('statusIN' + inProgressStatuses, 1);
          return executionGr.next() && executionGr;
      },

      getModelSnapshot: function() {
          if (NLUBatchTestDefinition.modelSnapshot)
              return NLUBatchTestDefinition.modelSnapshot;

          var modelIdsStr = this.getGR().getValue(FIELDS.MODELS);
          var modelIds = modelIdsStr ? modelIdsStr.split(',') : [];
          var isOptimize = this.getGR().getValue(FIELDS.IS_OPTIMIZE);
          return NLUBatchTestDefinition.validateAndGetModelSnapshot(modelIds, (isOptimize != 0 && !!isOptimize));
      },

      getTestSetSnapshot: function() {
          var testSet = new NLUBatchTestSet(this.getGR().test_set);
          return {
              last_updated_on: testSet.getLastUpdatedOn(),
              records_count: testSet.getUtteranceCount()
          };
      },

      getExecutionGr: function(filter, limit) {
          var executionGr = new GlideRecord(tables.NLU_BATCH_TEST_RUN_EXECUTION);
          executionGr.addQuery('test_run_definition', this.definitionId);
          if (filter) executionGr.addEncodedQuery(filter);
          executionGr.orderByDesc('sys_updated_on');
          if (limit) executionGr.setLimit(limit);
          executionGr.query();
          return executionGr;
      },

      purgeExecutions: function() {
          var maxRuns = parseInt(gs.getProperty(sysProps.MAX_BATCH_TEST_RUNS, constants.DEFAULT_MAX_RUNS));
          if (maxRuns < 0) return; // if value is 0 / -ve, then do not purge.

          var executionGr = this.getExecutionGr();
          var count = 1;
          while (executionGr.next()) {
              if (count > maxRuns) {
                  gs.info('Purging Batch Test execution: ' + executionGr.getUniqueValue());
                  executionGr.deleteRecord();
              }
              count++;
          }
      },

      type: 'NLUBatchTestDefinition'
  };

})();

Sys ID

b5b5ed6d0718201028ef0a701ad30079

Offical Documentation

Official Docs: