Name
global.ApplicationsFingerprintUpdateUtil
Description
Util class for Application fingerprint ML API
Script
var ApplicationsFingerprintUpdateUtil = Class.create();
ApplicationsFingerprintUpdateUtil.prototype = {
PRE_FINGERPRINT_KEY_PREFIX: "pre_app_fingerprint_update",
FINGERPRINT_KEY_PREFIX: "app_fingerprint_update",
SOLUTION_NAME_KEY: "app_fingerprint_solution_name",
TRAINING_COMPLETE_KEY: "app_fingerprint_training_complete",
SOLUTION_SUFFIX: "application_suggestion",
MIN_PROCESS_TO_TRAIN_PROPERTY: "process.cluster.batch_size.min_process_to_train",
MIN_PROCESS_TO_TRAIN_DEFAULT: "1000",
MAX_DAYS_BETWEEN_TRAIN_PROPERTY: "process.cluster.batch_size.max_days_between_train",
MAX_DAYS_BETWEEN_TRAIN_DEFAULT: "3",
initialize: function() {
this.smMLTableUtils = new SMMLTableUtils();
},
createSolution: function() {
try {
var label = this.SOLUTION_SUFFIX;
var fromDate;
var firstTime = false;
var key = this.FINGERPRINT_KEY_PREFIX;
var saHash = this.smMLTableUtils.getSaHashGR(key);
if (saHash) {
fromDate = saHash.getValue("hash");
} else {
gs.debug("ApplicationsFingerprintUpdateUtil: Could not find " + key + " in sa_hash table, taking the first row on v_cmdb_running_process");
fromDate = this.smMLTableUtils.getFirstCreatedInTable("cmdb_running_process", "sys_created_on", null);
if (JSUtil.nil(fromDate))
gs.warn("ApplicationsFingerprintUpdateUtil-->getFirstCreatedInTable: Could not find the first value, table " + table + " seems to be empty");
firstTime = true;
}
var maxProcessToGet = gs.getProperty("glide.app.fingerprint.maxprocess.update", 50000);
var toDate = this.smMLTableUtils.getToDate("v_cmdb_running_process", "sys_created_on", fromDate, maxProcessToGet, null, false);
// save the toDate in order for it to be used in the next iteration, pre is because it cant be used until batching is success
var filter = '';
if (toDate) {
filter = this.smMLTableUtils.getTableFilter("v_cmdb_running_process", "sys_created_on", fromDate, toDate, firstTime, null, false);
}
var myData = new sn_ml.DatasetDefinition({
'tableName': 'v_cmdb_running_process',
'fieldNames': ['cluster_name', 'cluster_parameters'],
'encodedQuery': filter
});
var mySolution = new sn_ml.ClusteringSolution({
'label': label,
'dataset': myData,
'inputFieldNames': ['cluster_parameters'],
'groupByFieldName': 'cluster_name',
'clusterConcept': 'tfidf',
'updateFrequency': 'do_not_update',
'trainingFrequency': 'run_once',
'processingLanguage': 'en',
'stopwords': ['Default English Stopwords'],
'advancedParams': {
'LD': '',
'Minimum Neighbors': 1,
'Epsilon': 0.25,
'DBSCAN': '',
'mluc': 'MLUC-ITOM-00001'
},
'scope': 'global'
});
// add solution
var createdSolutionName = sn_ml.ClusteringSolutionStore.add(mySolution);
this.smMLTableUtils.updateSaHash(this.SOLUTION_NAME_KEY, createdSolutionName);
// add mlMLJob to db and submit training job
var solutionVersion = mySolution.submitTrainingJob();
if (toDate) {
// we found a to_date, update it for next iteration
this.smMLTableUtils.updateSaHash(this.PRE_FINGERPRINT_KEY_PREFIX, toDate);
}
var trainingStatus = JSON.parse(solutionVersion.getStatus());
gs.debug("ApplicationsFingerprintUpdateUtil: Solution " + createdSolutionName + " was created, state is " + trainingStatus['state']);
return createdSolutionName;
} catch (ex) {
gs.error('ApplicationsFingerprintUpdateUtil: failed to create solution ' + solutionName + " exception:" + ex.getMessage());
return null;
}
},
updateSolution: function(solutionVersion, solutionName) {
if (this.smMLTableUtils.isUpdateComplete(solutionVersion, this.TRAINING_COMPLETE_KEY)) {
// set flag for training complete
this.smMLTableUtils.updateSaHash(this.TRAINING_COMPLETE_KEY, "true");
// if the status is success, make sure that the last run 'to' date is being set to from;
this.smMLTableUtils.updateFromDateOnSuccess(this.PRE_FINGERPRINT_KEY_PREFIX, this.FINGERPRINT_KEY_PREFIX);
}
var updateStatus = solutionVersion.getUpdateStatus();
// get inside if success or some kind of error happened
//if (updateStatus != "Waiting for Updating" && updateStatus != "Updating Solution" && updateStatus != "Fetching Files for Updating" && updateStatus != "Uploading Updated Solution") {// best case is "Update Complete"
try {
gs.debug("ApplicationsFingerprintUpdateUtil: current status before update is " + updateStatus);
var options = {};
var fromDate;
var key = this.FINGERPRINT_KEY_PREFIX;
var saHash = this.smMLTableUtils.getSaHashGR(key);
var firstTime = false;
if (saHash) {
fromDate = saHash.getValue("hash");
} else {
gs.debug("ApplicationsFingerprintUpdateUtil: Could not find " + key + " in sa_hash table, taking the first row on v_cmdb_running_process");
fromDate = this.smMLTableUtils.getFirstCreatedInTable("cmdb_running_process", "sys_created_on", null);
if (JSUtil.nil(fromDate))
gs.warn("ApplicationsFingerprintUpdateUtil-->getFirstCreatedInTable: Could not find the first value, table " + table + " seems to be empty");
firstTime = true;
}
if (fromDate) {
var maxProcessToGet = gs.getProperty("glide.app.fingerprint.maxprocess.update", 100000);
var toDate = this.smMLTableUtils.getToDate("v_cmdb_running_process", "sys_created_on", fromDate, maxProcessToGet, null, false);
// save the toDate in order for it to be used in the next iteration, pre is because it cant be used until batching is success
if (toDate) {
options.filter = this.smMLTableUtils.getTableFilter("v_cmdb_running_process", "sys_created_on", fromDate, toDate, firstTime, null, false);
options.table = "v_cmdb_running_process";
if(this.checkMinimumRequirements(options, solutionName)){
solutionVersion.submitUpdateJob(options);
// we found a to_date, update it for next iteration
this.smMLTableUtils.updateSaHash(this.PRE_FINGERPRINT_KEY_PREFIX, toDate);
gs.debug("ApplicationsFingerprintUpdateUtil: toDate is updated to " + toDate);
gs.debug("ApplicationsFingerprintUpdateUtil: Solution " + solutionName + " was submitted update, status:" + solutionVersion.getUpdateStatus());
} else {
gs.debug("ApplicationsFingerprintUpdateUtil: minimum condition for training are not met");
}
} else {
// in case we dont find a to_date, we are in the end of the list
gs.debug("ApplicationsFingerprintUpdateUtil: we have no more new items in the table for solution " + solutionName);
}
} else {
gs.debug("ApplicationsFingerprintUpdateUtil: Solution " + solutionName + " was skipped from update, no 'from date' was found, processes table might be empty");
}
} catch (e) {
gs.warn("ApplicationsFingerprintUpdateUtil: Solution " + solutionName + " is not in status 'Update Complete' but '" + updateStatus + "', skipped update, " + e);
}
},
/***
*
* @param solutionName
*/
submitUpdate: function() {
var solutionName = this.smMLTableUtils.getSolutionName(this.SOLUTION_NAME_KEY);
var existSolutionName;
var mlSolution;
try {
mlSolution = sn_ml.ClusteringSolutionStore.get(solutionName);
existSolutionName = solutionName;
} catch (e) {
gs.debug("Could not find solution " + solutionName);
}
if (mlSolution) {
var solutionVersion;
try {
solutionVersion = mlSolution.getActiveVersion();
} catch (e) {
gs.error("ApplicationsFingerprintUpdateUtil: Failed to get active version for solution " + solutionName + ", " + e);
return existSolutionName;
}
if (solutionVersion) {
this.updateSolution(solutionVersion, solutionName);
} else {
gs.warn("ApplicationsFingerprintUpdateUtil: No active version was found for Solution " + solutionName);
}
} else {
gs.debug("ApplicationsFingerprintUpdateUtil: Failed to find solution " + solutionName + ", creating and training the solution");
if (this.smMLTableUtils.isSolutionCreated(this.SOLUTION_NAME_KEY, this.SOLUTION_SUFFIX)) {
gs.warn("ApplicationsFingerprintUpdateUtil: Solution with name " + solutionName + " was already found, avoiding to create new solution");
} else {
existSolutionName = this.createSolution();
gs.debug("ApplicationsFingerprintUpdateUtil: Solution " + existSolutionName + ", was created and started training");
}
}
return existSolutionName;
},
checkMinimumRequirements: function(option, solutionName){
// Get Properties
var minProcessToTrain = GlideProperties.get(this.MIN_PROCESS_TO_TRAIN_PROPERTY, this.MIN_PROCESS_TO_TRAIN_DEFAULT);
var maxDayBetweenTrain = GlideProperties.get(this.MAX_DAYS_BETWEEN_TRAIN_PROPERTY, this.MAX_DAYS_BETWEEN_TRAIN_DEFAULT);
// Check if there is minimum record from the relevant table.
var grOption = new GlideRecord(option.table);
grOption.addQuery(option.filter);
grOption.query();
if (grOption.getRowCount() < minProcessToTrain){
// check time elapsed from last running date.
var grSolution = new GlideRecord("ml_solution");
grSolution.addQuery("solution_name", solutionName);
grSolution.query();
if (grSolution.next()){
var solutionLastRuntime = grSolution.getValue("sys_updated_on");
var gdtNow = new GlideDateTime();
var gdtLastRuntime = new GlideDateTime(solutionLastRuntime);
gdtLastRuntime.addDays(maxDayBetweenTrain);
// If both requirement not met - return false.
if(gs.dateDiff (gdtLastRuntime, gdtNow, true) < 0) return false;
}
}
return true;
},
type: 'ApplicationsFingerprintUpdateUtil'
};Sys ID
d53d2c67736300105ad2db37aef6a70c