Name

global.MLSchemaFixScriptHelper

Description

No description available

Script

var MLSchemaFixScriptHelper = Class.create();
MLSchemaFixScriptHelper.prototype = {
  initialize: function() {
  	this.prev_sol_def_map = {};
  	this.new_sol_def_map = {};
  	this.mapPrevandNewSolutionDef();
  },
  /*
  * Helper funct to move solution definition from ml_solution_definition table to new schema
  * input - solutionsArr ( Array of Solution Names)
  *       - type (include / exclude)
  *
  */
  fixMLSchemaforOrlandoUpgrade: function(solutionsArr, type){
  	gs.info('MLinfo - Started moving solution definition to new schema');
  	var similarity_capability = "15ab7f3c53873300d1dcddeeff7b12ce";
  	var clustering_capability = "366d333453c73300d1dcddeeff7b12fc";
  	var classification_capability = "95ca97b453873300d1dcddeeff7b120f";
  	var nlu_capability = "b4ddf73453c73300d1dcddeeff7b1288";
  	var eng_stopwords = "a6282a25533333008c5bddeeff7b12c3";
  	var ml_def_gr = new GlideRecord('ml_solution_definition');
  	if (type == "include" && !JSUtil.nil(solutionsArr) && solutionsArr.toString())
  		ml_def_gr.addQuery('solution_name','IN',solutionsArr.toString());
  	ml_def_gr.query();
  	while (ml_def_gr.next()) {
  		// do not run the fix script for nlu oob models as the updated ml_capability_definition_base records
  		// are present in the update folders of respective plugins and need to be applied as part of that plugin
  		// update. If this fix script runs before the plugin update, it creates a new ml_capability_definition_base
  		// record which has a different sys_id than the one present in the plugin's update folder
  		if (type == "exclude" && !JSUtil.nil(solutionsArr) && JSUtil.contains(solutionsArr, ml_def_gr.solution_name)) {
  			gs.info('Ignore the Solution ' + ml_def_gr.solution_name);
  			continue;
  		}
  		gs.info('Transfer the Solution ' + ml_def_gr.solution_name);
  		var trainer = ml_def_gr.trainer.name + '';
  		var new_ml_def_gr = '';
  		
  		//OOB solution definition should be updated if the customer has made changes to it.
  		var is_solution_def_present = this.isSolutionDefinitionPresent(ml_def_gr.solution_name);
  		var is_solution_def_cust_updated = this.isSolutionDefinitionCustomerUpdate(ml_def_gr.getUniqueValue());
  		if (is_solution_def_present && !is_solution_def_cust_updated ){
  			gs.info("solution definition "+ ml_def_gr.solution_name +" is present in new schema and it is not customer updated");
  			continue;
  		}
  		switch(trainer) {
  			case 'similarity_trainer':
  				new_ml_def_gr = new GlideRecord('ml_capability_definition_similarity');
  				if (is_solution_def_present) {
  					new_ml_def_gr.addQuery('solution_name',ml_def_gr.solution_name);
  					new_ml_def_gr.query();
  					if (!new_ml_def_gr.next()) {
  						new_ml_def_gr.initialize();
  					}
  				} else {
  					new_ml_def_gr.initialize();
  				}

  				// Upgrading from madrid to orlando word vector for similarity is empty so create a new word vector and add to capability definition
  				if (ml_def_gr.getValue('word_vector_corpus')) {
  					new_ml_def_gr.setValue('word_vector_corpus',ml_def_gr.getValue('word_vector_corpus'));
  				} else {
  					var RecordInfo = {};
  					RecordInfo.inputTableName = ml_def_gr.getValue('table');
  					//gr.getValue(condition_field) returns null for empty condition field. gr.setValue(condition_field,null) doesnot work 
  					//need to use gr.setValue(condition_field,"");
  					RecordInfo.inputTableFilter = JSUtil.nil(ml_def_gr.getValue('input_filter')) ? "": ml_def_gr.getValue('input_filter');
  					RecordInfo.inputFields = ml_def_gr.getValue('input_fields');
  					RecordInfo.predictionInputFields = ml_def_gr.getValue('prediction_input_fields');
  					RecordInfo.domain = ml_def_gr.getValue('sys_domain');
  					RecordInfo.solution_name = ml_def_gr.getValue('solution_name');
  					var wordVectorSysId = this.createWordVectorCorpus(RecordInfo);
  					new_ml_def_gr.setValue('word_vector_corpus',wordVectorSysId);
  				}
  				if (ml_def_gr.getValue('table_to_compare')) {
  					new_ml_def_gr.setValue('test_table', ml_def_gr.getValue('table_to_compare'));
  					new_ml_def_gr.setValue('test_fields', ml_def_gr.getValue('fields_to_compare'));
  				} else {
  					new_ml_def_gr.setValue('test_table', ml_def_gr.getValue('table'));
  					new_ml_def_gr.setValue('test_fields', ml_def_gr.getValue('prediction_input_fields'));
  				}
  				new_ml_def_gr.setValue('threshold', ml_def_gr.trainer.default_threshold);
  				new_ml_def_gr.setValue('fields', ml_def_gr.getValue('prediction_input_fields'));
  				var similarity_window_filter = JSUtil.nil(ml_def_gr.getValue('similarity_window_filter')) ? "": ml_def_gr.getValue('similarity_window_filter');
  				new_ml_def_gr.setValue('filter', similarity_window_filter);
  				new_ml_def_gr.setValue('update_frequency', ml_def_gr.getValue('update_frequency'));
  				new_ml_def_gr.setValue('capability',similarity_capability);
  				break;
  			case 'classification_trainer':
  				new_ml_def_gr = new GlideRecord('ml_capability_definition_classification');
  				if (is_solution_def_present) {
  					new_ml_def_gr.addQuery('solution_name',ml_def_gr.solution_name);
  					new_ml_def_gr.query();
  					if (!new_ml_def_gr.next()) {
  						new_ml_def_gr.initialize();
  					}
  				} else {
  					new_ml_def_gr.initialize();
  				}
  				new_ml_def_gr.setValue('output_field', ml_def_gr.getValue('output_field'));
  				new_ml_def_gr.setValue('fields', ml_def_gr.getValue('input_fields'));
  				var filter = JSUtil.nil(ml_def_gr.getValue('input_filter')) ? "": ml_def_gr.getValue('input_filter');
  				new_ml_def_gr.setValue('filter', filter);
  				new_ml_def_gr.setValue('capability',classification_capability);
  				break;
  			case 'clustering_trainer':
  				new_ml_def_gr = new GlideRecord('ml_capability_definition_clustering');
  				if (is_solution_def_present) {
  					new_ml_def_gr.addQuery('solution_name',ml_def_gr.solution_name);
  					new_ml_def_gr.query();
  					if (!new_ml_def_gr.next()) {
  						new_ml_def_gr.initialize();
  					}
  				} else {
  					new_ml_def_gr.initialize();
  				}
  				new_ml_def_gr.setValue('word_vector_corpus',ml_def_gr.getValue('word_vector_corpus'));
  				new_ml_def_gr.setValue('fields', ml_def_gr.getValue('prediction_input_fields'));
  				var similarity_window_filter = JSUtil.nil(ml_def_gr.getValue('similarity_window_filter')) ? "": ml_def_gr.getValue('similarity_window_filter');
  				new_ml_def_gr.setValue('filter', similarity_window_filter);
  				new_ml_def_gr.setValue('update_frequency', ml_def_gr.getValue('update_frequency'));
  				new_ml_def_gr.setValue('min_records', ml_def_gr.getValue('min_records'));
  				if (ml_def_gr.getValue("use_segmentation") == 'true'){
  					new_ml_def_gr.setValue('use_segmentation', ml_def_gr.getValue('use_segmentation'));
  					new_ml_def_gr.setValue('segmentation_field', ml_def_gr.getValue('segmentation_field'));
  				}
  				new_ml_def_gr.setValue('capability',clustering_capability);
  				break;
  			case 'nlu_trainer':
  				new_ml_def_gr = new GlideRecord('ml_capability_definition_base');
  				if (is_solution_def_present) {
  					new_ml_def_gr.addQuery('solution_name',ml_def_gr.solution_name);
  					new_ml_def_gr.query();
  					if (!new_ml_def_gr.next()) {
  						new_ml_def_gr.initialize();
  					}
  				} else {
  					new_ml_def_gr.initialize();
  				}
  				new_ml_def_gr.setValue('capability',nlu_capability);
  				break;
  		}
  		var new_ml_def_gr_sys_id = "";
  		if(new_ml_def_gr){
  			new_ml_def_gr.setValue('solution_label', ml_def_gr.getValue('solution_label'));
  			new_ml_def_gr.setValue('solution_name', ml_def_gr.getValue('solution_name'));
  			new_ml_def_gr.setValue('active', ml_def_gr.getValue('active'));
  			new_ml_def_gr.setValue('sys_domain', ml_def_gr.getValue('sys_domain'));
  			new_ml_def_gr.setValue('sys_scope', ml_def_gr.getValue('sys_scope'));
  			new_ml_def_gr.setValue('table', ml_def_gr.getValue('table'));
  			new_ml_def_gr.setValue('dataset_language', ml_def_gr.getValue('dataset_language'));
  			new_ml_def_gr.setValue('training_frequency', ml_def_gr.getValue('training_frequency'));
  			new_ml_def_gr.setValue('current_solution_version', ml_def_gr.getValue('current_solution_version'));
  			new_ml_def_gr.setValue('stopwords',eng_stopwords);
  			new_ml_def_gr.setWorkflow(false);
  			
  			if (is_solution_def_present) { 
  				new_ml_def_gr_sys_id = new_ml_def_gr.update();	
  			} else {
  				new_ml_def_gr_sys_id = new_ml_def_gr.insert();
  			}
  			var checkgr = new GlideRecord('ml_capability_definition_base');
  			checkgr.addQuery('solution_name',new_ml_def_gr.getValue("solution_name"));
  			checkgr.query();
  			if (checkgr.next() && (checkgr.getValue("sys_scope") != ml_def_gr.getValue('sys_scope'))){
  				SNC.MLFixScriptUtil.updateCapabilitySolDefScope(new_ml_def_gr.getValue("solution_name"));
  			}
  			new_ml_def_gr.setWorkflow(true);
  		}

  		if (new_ml_def_gr_sys_id) {

  			// connect all the solutions to new capability definition
  			var solution_gr = new GlideRecord('ml_solution');
  			solution_gr.addQuery('solution_definition', ml_def_gr.getUniqueValue());
  			solution_gr.query();
  			while ( solution_gr.next() ) {
  				//table_to_compare and fields_to_compare will be empty for pre Newyork solutions
  				//Adding values to use test solutions on pre newyork  traineds solutions
  				if (trainer == 'similarity_trainer' && JSUtil.nil(solution_gr.getValue("table_to_compare"))){
  					solution_gr.setValue('table_to_compare', solution_gr.getValue("table"));
  				}
  				if (trainer == 'similarity_trainer' && JSUtil.nil(solution_gr.getValue("fields_to_compare"))){
  					solution_gr.setValue('fields_to_compare',solution_gr.getValue("prediction_input_fields"));
  				}
  				if (JSUtil.nil(solution_gr.getValue("capability"))){
  					solution_gr.setValue('capability',trainer);
  				}
  				solution_gr.setValue('ml_capability_definition', new_ml_def_gr_sys_id);
  				solution_gr.setWorkflow(false);
  				solution_gr.update();
  				solution_gr.setWorkflow(true);
  			}
  			// update advanced solution parameters for clustering solution
  			if(trainer == 'clustering_trainer') {
  				//target solution coverage, minimum number of records per cluster
  				var clustering_params = ['999c201453733300d1dcddeeff7b1212'];
  				for (var i=0; i< clustering_params.length; i++){
  					var gr = new GlideRecord('ml_advanced_solution_settings');
  					gr.initialize();
  					gr.setValue('solution_parameters', clustering_params[i]);
  					gr.setValue('ml_capability_definition', new_ml_def_gr_sys_id);
  					gr.setValue('user_inputs',ml_def_gr.getValue('threshold'));
  					gr.insert();
  				}
  			}
  			//update all the training request schedule to new capability solution
  			var training_schedule_gr = new GlideRecord('ml_training_request_schedule');
  			training_schedule_gr.addQuery('solution_definition',ml_def_gr.getUniqueValue());
  			training_schedule_gr.query();
  			while (training_schedule_gr.next()){
  				training_schedule_gr.setValue('ml_capability_definition', new_ml_def_gr_sys_id);
  				training_schedule_gr.setWorkflow(false);
  				training_schedule_gr.update();
  				training_schedule_gr.setWorkflow(true);
  			}

  			//update all the update request schedule to new capability solution
  			var update_schedule_gr = new GlideRecord('sysauto_script');
  			update_schedule_gr.addQuery('name',ml_def_gr.getUniqueValue());
  			update_schedule_gr.query();
  			while (update_schedule_gr.next()){
  				update_schedule_gr.setValue('name', new_ml_def_gr_sys_id);
  				update_schedule_gr.setWorkflow(false);
  				update_schedule_gr.update();
  				update_schedule_gr.setWorkflow(true);
  			}
  		}
  	}
  },
  createWordVectorCorpus: function(similarityRecordInfo){
  	// Create a word vector corpus record
  	var wordVecCorpusGR = new GlideRecord('ml_word_vector_corpus');
  	wordVecCorpusGR.initialize();
  	wordVecCorpusGR.setValue('name', similarityRecordInfo.solution_name);
  	wordVecCorpusGR.setValue('active', true);
  	wordVecCorpusGR.setValue('version', '1');
  	wordVecCorpusGR.setValue('sys_domain', similarityRecordInfo.domain);
  	var wordVecCorpusGRSysId = wordVecCorpusGR.insert();
  	/*
  	 * Create a word vector corpus details reocrd and add the word vector corpus record as
  	 * a reference
  	 */
  	var wordVecCorpusDetailsGR = new GlideRecord('ml_word_vector_corpus_details');
  	wordVecCorpusDetailsGR.initialize();
  	wordVecCorpusDetailsGR.setValue('name', similarityRecordInfo.solution_name + '_' + similarityRecordInfo.inputTableName);
  	wordVecCorpusDetailsGR.setValue('table', similarityRecordInfo.inputTableName);
  	wordVecCorpusDetailsGR.setValue('filter', similarityRecordInfo.inputTableFilter);
  	wordVecCorpusDetailsGR.setValue('field_list', similarityRecordInfo.inputFields);
  	wordVecCorpusDetailsGR.setValue('sys_domain', similarityRecordInfo.domain);
  	wordVecCorpusDetailsGR.setValue('word_vector_corpus',wordVecCorpusGRSysId);
  	wordVecCorpusDetailsGR.insert();
  	return wordVecCorpusGRSysId;
  },
  isSolutionDefinitionCustomerUpdate: function(solution_rec_id){
  	var gr = new GlideRecord("sys_update_xml");
  	gr.addQuery("name","ml_solution_definition_"+solution_rec_id);
  	gr.query();
  	if (gr.next()){
  		return true;
  	}
  	return false;
  },
  isSolutionDefinitionPresent: function(solution_name){
  	var gr = new GlideRecord("ml_capability_definition_base");
  	gr.addQuery('solution_name',solution_name);
  	gr.query();
  	if(gr.next()){
  		return true;
  	}
  	return false;
  },
  /*
  * input:
  *       Table - which has reference to ml_solution_definition
  *       previous column
  *       new column
  * usage - new MLSchemaFixScriptHelper().
  * migrateMLDefinitionReferences('ml_solution','solution_definition','ml_capability_definition'));
  */
  migrateMLDefinitionReferences: function(table, previous_col, new_col){
  	if (JSUtil.nil(table) || JSUtil.nil(previous_col) || JSUtil.nil(new_col)){
  		return;
  	}
  	var gr = new GlideRecord(table);
  	if (!gr.isValid()){
  		return;
  	}
  	gs.info("Updating solution definition reference for table "+table);
  	gr.query();
  	while(gr.next()){
  		var solution_name = this.prev_sol_def_map[ gr.getValue( previous_col )];
  		var new_solution_sysid = this.new_sol_def_map[solution_name];
  		if (new_solution_sysid && JSUtil.nil(gr.getValue(new_col))) {
  			gr.setValue(new_col, new_solution_sysid);
  			gr.update();
  			gs.info("Updating reference for "+solution_name);
  		}
  	}
  },
  /*
  	previous and new solution definition have same solution_name.
  	Solution name is unique across all the solution definitions
  */
  mapPrevandNewSolutionDef: function(){
  	var prev_ml_def_gr = new GlideRecord('ml_solution_definition');
  	prev_ml_def_gr.query();
  	while (prev_ml_def_gr.next()){
  		this.prev_sol_def_map[prev_ml_def_gr.getUniqueValue()] = prev_ml_def_gr.getValue('solution_name');
  	}
  	var new_ml_def_gr = new GlideRecord('ml_capability_definition_base');
  	new_ml_def_gr.query();
  	while (new_ml_def_gr.next()){
  		this.new_sol_def_map[new_ml_def_gr.getValue('solution_name')] = new_ml_def_gr.getUniqueValue();
  	}
  },
  type: 'MLSchemaFixScriptHelper'
};

Sys ID

7d884cdd53540010d1dcddeeff7b122c

Offical Documentation

Official Docs: