Name

sn_nlu_workbench.NLUBatchTestProcessor

Description

Utilities to process the batch test set data and upload to utterance table

Script

var NLUBatchTestProcessor = Class.create();

var constants = NLUWorkbenchConstants.constants;
var tables = NLUWorkbenchConstants.tables;

function getOnStartScript(testSetId) {
  return "(function runTransformScript(source, map, log, target) { \
  var importSetId = import_set.getUniqueValue(); \
  var dataSource = import_set.getValue('data_source'); \
  var updatedData = { properties: 'charset=utf-8' }; \
  global.NLUSystemUtil.updateOrInsert('sys_data_source', 'sys_id', dataSource, updatedData); \
  var gr = new GlideRecord('" + tables.NLU_BATCH_TEST_SET + "'); \
  if (gr.get('" + testSetId + "')) { \
      gr.setValue('status', 'uploading'); \
      gr.setValue('import_set', importSetId); \
      gr.update(); \
  } \
})(source, map, log, target);";
}

function getOnBeforeScript(testSetId) {
  return "(function runTransformScript(source, map, log, target) { \
  var expectedintent = source.getValue('u_expected_intent');\
  if(expectedintent != null && expectedintent.toLowerCase() == 'not relevant') { \
  	source.setValue('u_expected_intent',''); \
  	expectedintent = '';\
  } \
  if(typeof expectedintent === 'string' && expectedintent.split(',').length > " + constants.MAX_EXPECTED_INTENTS + ") { \
  	ignore = true; \
  }\
  target.test_set = '" + testSetId + "'; \
})(source, map, log, target);";
}

function getOnBeforeScriptDefaultTestset(testSetId, modelIntents) {
  return "(function runTransformScript(source, map, log, target) { \
  var expectedintent = source.getValue('u_expected_intent');\
  target.test_set = '" + testSetId + "'; \
  if(expectedintent != null && expectedintent.toLowerCase() == 'not relevant') { \
      source.setValue('u_expected_intent',''); \
      expectedintent = '';\
  } \
  var modelIntents = " + JSON.stringify(modelIntents) + "\
  ignore = NLUBatchTestSet.canIgnoreUtterance(expectedintent, modelIntents);\
  if (!ignore) {\
      var expectedIntentsArr = NLUBatchTestSet.getMappedModelIntents(expectedintent, modelIntents); \
      source.setValue('u_expected_intent', expectedIntentsArr.join(',')); \
  }\
})(source, map, log, target);";
}

function getOnCompleteScript(testSetId) {
  return "(function runTransformScript(source, map, log, target) { \
  var gr = new GlideRecord('" + tables.NLU_BATCH_TEST_SET + "'); \
  if (gr.get('" + testSetId + "')) { \
      gr.setValue('status', error ? 'upload_failed' : 'uploaded'); \
      gr.update(); \
  } \
})(source, map, log, target);";
}

(function() {
  NLUBatchTestProcessor.createBatchTestSet = function(data) {
      try {
          var fileName = data.file && data.file.name;
          var fileSize = data.file && data.file.size;
          var testSetName = data.name;
          var testSetLanguage = data.language;
          var testSetId = data.testSetId;
          var associatedModel = data.model;

          // Basic validations of file:
          if (data.file) {
              if (!NLUFileUtil.checkFileSize(fileSize)) {
                  throw new Error(gs.getMessage('File size exceeds max filesize of {0} MB', NLUFileUtil.getMaxFileSize()));
              }
              var fileArr = NLUFileUtil.getFileNameAndExtension(fileName, constants.VALID_FILE_TYPES);
              if (!fileArr) {
                  throw new Error(gs.getMessage('Please upload a file with a valid extension'));
              }
              var fileNameWithoutExt = fileArr[0];
          }

          // Create record in nlu_batch_test_set
          if (!testSetId) {
              testSetId = NLUBatchTestSet.addRecord(
                  testSetName || fileNameWithoutExt,
                  testSetLanguage || constants.DEFAULT_LANGUAGE, !data.file && 'uploaded', associatedModel);
              if (!testSetId) {
                  throw new Error(gs.getMessage('Failed to create a batch test set record'));
              }
          }

          if (!data.file) {
              return {
                  status: 'success',
                  testSetId: testSetId,
              };
          }

          // Create staging table for import set
          var fields = [{
                  name: 'expected_intent',
                  length: 500
              },
              {
                  name: 'utterance',
                  length: 512
              }
          ];
          var stageTableStructure = NLUImportSetUtils.createStageTable('NLU Batch Test Stage Table', fields);
          if (!stageTableStructure) {
              throw new Error(gs.getMessage('Failed to create a staging table for import set'));
          }

          // Create & associate a transform map for the stage table
          var mapSysId = NLUImportSetUtils.createTransformMap('NLU Batch Test Transform Map',
              stageTableStructure, tables.NLU_BATCH_TEST_UTTERANCE, testSetId, {
                  u_expected_intent: 'intent'
              });
          if (!mapSysId) {
              throw new Error(gs.getMessage('Failed to create transform map for staging table: {0}', stageTableStructure['tableName']));
          }
          NLUImportSetUtils.createTransformMapScripts(mapSysId, [{
                  event: 'onStart',
                  script: getOnStartScript(testSetId)
              },
              {
                  event: 'onBefore',
                  script: associatedModel ?
                      getOnBeforeScriptDefaultTestset(testSetId, NLUBatchTestSet.getModelIntentsFromTestId(testSetId)) : getOnBeforeScript(testSetId)
              },
              {
                  event: 'onComplete',
                  script: getOnCompleteScript(testSetId)
              }
          ], 'sn_nlu_workbench');

          return {
              status: 'success',
              testSetId: testSetId,
              stageTable: stageTableStructure['tableName'],
              transformMap: mapSysId
          };
      } catch (e) {
          return {
              status: 'failure',
              message: e.message
          };
      }
  };

  NLUBatchTestProcessor.prototype = {
      initialize: function() {},
      type: 'NLUBatchTestProcessor'
  };

})();

Sys ID

116030740700201028ef0a701ad3003b

Offical Documentation

Official Docs: