Name

sn_nlu_workbench.NLUBatchTestIntegrator

Description

Integrater script for running batch test & get status

Script

var NLUBatchTestIntegrator = Class.create();

(function() {

  var constants = NLUWorkbenchConstants.constants;
  var trainingMode = NLUWorkbenchConstants.TRAINING_MODE;

  NLUBatchTestIntegrator.addSolution = function(batchTestSetName, isOptimized) {
      gs.info('NLU Batch Test - adding solution : ' + batchTestSetName);
      try {
          var solutionInfo = {
              nluTrainingMode: isOptimized ? trainingMode.TRAIN_OPTIMIZE : trainingMode.ASYNC_BATCH,
              label: batchTestSetName
          };

          var solution = new sn_ml.NLUSolution(solutionInfo, {
              dummyKey: 'dummyValue'
          });
          return sn_ml.NLUSolutionStore.add(solution);
      } catch (e) {
          gs.error('NLU Batch Test - Failed to add solution: ' + e.message);
          return null;
      }
  };

  NLUBatchTestIntegrator.getTrainingMode = function(mlSolutionGr) {
      var mlDefinition = mlSolutionGr.ml_capability_definition;
      var mlProps = mlDefinition && mlDefinition.solution_properties;
      return JSON.parse(mlProps || {}).nluTrainingMode;
  };

  NLUBatchTestIntegrator.isTrainingModeValid = function(mlSolutionGr) {
      try {
          var mode = NLUBatchTestIntegrator.getTrainingMode(mlSolutionGr);
          return [trainingMode.ASYNC_BATCH, trainingMode.TRAIN_OPTIMIZE].indexOf(mode) >= 0;
      } catch (e) {
          return false;
      }
  };

  NLUBatchTestIntegrator.getOptimizeSupportedLanguages = function() {
      try {
          var optimizeSupportMap = {};
          var query = 'query { NLU { supportedLanguages { name, language, releaseFeatures {name, isSupported, nonSupportedVersions}}}}';
          var res = global.NLUParloIntegrator.getCapability(query);
          if (res && res.state === 'success' && !res.errors) {
              var supportedLanguages = (res.data && res.data.NLU && res.data.NLU.supportedLanguages) || [];
              for (var i = 0; i < supportedLanguages.length; i++) {
                  if (supportedLanguages[i].releaseFeatures) {
                      supportedLanguages[i].releaseFeatures.forEach(function(feat) {
                          if ((feat.name || '').toLowerCase() === 'optimize' && feat.isSupported === true) {
                              optimizeSupportMap[supportedLanguages[i].language] = {
                                  nonSupportedVersions: feat.nonSupportedVersions || [],
                                  label: supportedLanguages[i].name
                              };
                          }
                      });
                  }
              }
          }
          return {
              status: 'success',
              optimizeSupportMap: optimizeSupportMap
          };
      } catch (e) {
          return {
              status: 'failure',
              message: gs.getMessage('Error while fetching optimization support')
          };
      }
  };

  /**
   * Will execute batch test with given utterances on modelList
   * utterances: JSON object of format { "utterances": [ { "utterance": "..." }, .... ]
   * modelList: String of format "[[{solutionName:'ScriptnluModel', solutionVersion: 1}]]"
   *
   * Return: 
   * {
   *     status: <success | failure>
   *     solutionName: <>
   *     solutionVersion: <>
   * }
   */
  NLUBatchTestIntegrator.submitTestRun = function(solutionName, trainJson) {
      var output = {
          status: 'failure'
      };
      try {
          if (trainJson && trainJson.solutionInfo && trainJson.payload) {
              var solutionInfo = trainJson.solutionInfo;
              var payload = trainJson.payload;
              gs.info('NlU Batch Test - running : ' + JSON.stringify(solutionInfo));

              var solution = new sn_ml.NLUSolution(solutionInfo, payload);
              sn_ml.NLUSolutionStore.update(solutionName, solution);
              var solutionVersion = solution.submitTrainingJob({});
              if (!solutionVersion) throw new Error('Failed to submit train');

              var solutionStatus = solutionVersion.getStatus();
              if (!solutionStatus) throw new Error('Failed to get the status of train submission');

              solutionStatus = JSON.parse(solutionStatus);
              output.status = solutionStatus.hasJobEnded === 'false' ? 'success' : 'failure';
              output.solutionVersion = solutionVersion && solutionVersion.getVersionNumber();
              output.solutionName = solutionName;
          } else {
              output.message = gs.getMessage('Invalid train json');
          }
      } catch (e) {
          gs.error('Error while submitting batch test run : ' + e.message);
          output.message = e.message;
      }
      gs.info('NlU Batch Test - execution job: ' + JSON.stringify(output));
      return output;
  };

  NLUBatchTestIntegrator.cancelTestRun = function(solutionName) {
      var result = {};
      try {
          var solution = sn_ml.NLUSolutionStore.get(solutionName);
          solution.cancelTrainingJob();
          result.status = 'success';
      } catch (e) {
          gs.debug('NLU Batch Test run cancel error' + e.message);
          result.status = 'failure';
          result.message = e.message;
      }
      return result;
  };

  NLUBatchTestIntegrator.copyArtifacts = function(fromSolutionName, fromSolutionVersion, toSolutionName) {
      var result = {
          status: 'failure'
      };
      try {
          var solution = sn_ml.NLUSolutionStore.get(fromSolutionName, {});
          var solutionVersion = solution.getVersion(fromSolutionVersion);
          if (solutionVersion && solutionVersion.copyArtifacts(toSolutionName)) {
              result.status = 'success';
          }
      } catch (e) {
          gs.debug('NLU Batch Test - Optimization apply failure: ' + e.message);
          result.message = e.message;
      }
      return result;
  };

  /**
   * Will get the results for solutionName, versionNumber
   * 
   * Result: 
   * {
   *     "status": "success",
   *     "response": {
   *         "inferences": [
   *             {
   *                 "utterance": "Jimmy should be added as my emergency contact",
   *                 "intents": [
   *                     {
   *                         "intentName": "AddEmergencyContact",
   *                         "nluModelName": "ScriptnluModel",
   *                         "score": 0.77628595
   *                     }
   *                 ]
   *             },
   *             ...
   *         ]
   *     }
   * }
   */
  NLUBatchTestIntegrator.getTestRunResults = function(solutionName, versionNumber) {
      try {
          var solutionVersion = NLUBatchTestIntegrator.getTestRunSolution(solutionName, versionNumber);

          var solutionVersionProperties = JSON.parse(solutionVersion.getProperties());
          var mode = NLUBatchTestIntegrator.getTrainingMode(
              NLUMLSolutionUtil.getMlGr(solutionName, versionNumber));
          var isOptimizedMode = (mode === trainingMode.TRAIN_OPTIMIZE);

          var testingResults = {};
          var summaryResults = {};
          var optimizedResults = null;
          var optimizedThreshold = {};

          try {
              if (isOptimizedMode) {
                  // getArtifactAsString is support for Rome and above only.
                  testingResults = solutionVersion.getArtifactAsString("current_predictions");

                  var hasValidationErrors = false;
                  var validationSummary = NLUBatchTestIntegrator.getTestsetValidationData(solutionVersion);
                  if (validationSummary.status === 'success') {
                      hasValidationErrors = validationSummary.response.dimensionIssues.some(function(issue) {
                          return issue.type === 'ERROR';
                      });
                  }
                  if (!hasValidationErrors) {
                      optimizedResults = JSON.parse(solutionVersion.getArtifactAsString("optimized_predictions"));
                      var authoringModel = solutionVersionProperties.authoringModel;
                      optimizedThreshold[authoringModel.name] = authoringModel.confidenceThreshold;
                  }
              } else {
                  testingResults = solutionVersion.getBatchTestingResult();
                  summaryResults = JSON.parse(solutionVersion.getBatchThresholdSummaryResult());
              }
          } catch (e) {
              gs.debug('NlU Batch Test - Recommendation or Optimisation not available: ' + e.message);
          }

          return {
              status: 'success',
              testing: JSON.parse(testingResults),
              summary: summaryResults,
              optimzed: optimizedResults,
              optimizedThreshold: optimizedThreshold
          };
      } catch (e) {
          gs.debug('NlU Batch Test - Error while getting results : ' + e.message);
          return {
              status: 'failure',
              message: e.message
          };
      }
  };

  NLUBatchTestIntegrator.getTestRunSolution = function(solutionName, versionNumber) {
      try {
          gs.debug('NlU Batch Test - Getting status for : ' + solutionName + ' (' + versionNumber + ')');
          var solution = sn_ml.NLUSolutionStore.get(solutionName);
          if (!solution) throw new Error('Solution does not exist with given name');

          var solutionVersion = solution.getVersion(versionNumber);
          if (!solutionVersion) throw new Error('Solution version not exist with given version number');

          var solutionStatus = JSON.parse(solutionVersion.getStatus(true));
          if (!solutionStatus || solutionStatus.status === 'failure')
              throw new Error(solutionStatus.message || 'Unable to get solution status');

          // Got the solution, but check if it's complete:
          if (solutionStatus.state !== 'solution_complete') throw new Error('Solution is still in progress');

          return solutionVersion;
      } catch (e) {
          gs.debug('NlU Batch Test - Error while getting status : ' + e.message);
          return {
              status: 'failure',
              message: e.message
          };
      }
  };

  NLUBatchTestIntegrator.getTestsetValidationData = function(solutionVersion) {
      try {
          var validationSummary = solutionVersion.getArtifactAsString("test_data_validation_summary");
          return JSON.parse(validationSummary);
      } catch (e) {
          return {
              status: 'failure',
              message: e.message
          };
      }
  };

  NLUBatchTestIntegrator.prototype = {
      initialize: function() {},
      type: 'NLUBatchTestIntegrator'
  };
})();

Sys ID

385715f00740201028ef0a701ad3008a

Offical Documentation

Official Docs: