Name
sn_app_insights.P1Model
Description
Script to predict P1 incidents periodically.
Script
var P1Model = Class.create();
P1Model.prototype = {
initialize: function() {
},
predict: function() {
var timeSeriesLength = 60;
var metrics = ['semaphores', 'sys_load'];
var minutesAgoStart = timeSeriesLength + 1; // add one because sometimes the last minute is cut off
var end = new GlideDateTime();
var start = new GlideDateTime(end);
start.addSeconds(-1 * minutesAgoStart * 60);
// query subject records
var grTable = new GlideRecord('sys_cluster_state');
grTable.query();
var grInsert = new GlideRecord('sn_app_insights_p1_predict');
grInsert.initialize();
var index = 0;
// load each metric time series into GlideRecord
for (key in metrics) {
metric = metrics[key];
index = this._getData(start, end, grTable, grInsert, metric, index, gs.getProperty('sn_app_insights.p1_predict_factor.' + metric), timeSeriesLength);
}
// If there is not sufficient data for a prediction we do nothing
if (index == metrics.length * timeSeriesLength) {
var sys_id = grInsert.insert();
this._predict(sys_id);
}
},
_getData: function(start, end, grTable, grInsert, metric, index, normalizer, length) {
// build transform, included below is fix for bug in metricbase
var resamplePeriodDuration;
var scheduledJob = new GlideRecord("sysauto_script");
scheduledJob.get("8509fd6cc7852010ff7667617ec26038");
resamplePeriodDuration = new GlideDuration(scheduledJob.run_period.dateNumericValue());
var transformer = new sn_clotho.Transformer(grTable);
transformer.metric(metric).resample("AVG", resamplePeriodDuration).avg();
// execute and return result for prediction
var data = transformer.execute(start, end).getData().getValues();
var validArray = data.filter(function(value) {
return !(isNaN(value) || value == null);
});
i = index;
for (var key in validArray) {
if (validArray.hasOwnProperty(key)) {
if (i == 0)
grInsert['u_' + i] = validArray[key] / normalizer;
else if (i - index < length)
grInsert['u_' + i + '_0'] = validArray[key] / normalizer;
else
break;
i++;
}
}
return i;
},
_predict: function(sys_id) {
if (sys_id !== null) {
var grPredict = new GlideRecord('sn_app_insights_p1_predict');
grPredict.get(sys_id);
var options = {};
var threshold = gs.getProperty('sn_app_insights.p1_predict_threshold', 90);
options.top_n = 1;
options.apply_threshold = false;
var predictionObject = JSON.parse(sn_ml.ClassificationSolutionStore.get('ml_x_global_global_p1_predict_script').getLatestVersion().predict(grPredict, options));
var prediction = predictionObject[grPredict.sys_id][0].predictedValue;
var confidence = parseFloat(predictionObject[grPredict.sys_id][0].confidence);
var maxCoolDown = gs.getProperty('sn_app_insights.p1_predict_max_cooldown', 5);
var state = this._getState(maxCoolDown);
if (prediction === "true" && confidence >= threshold) {
if (state === false) {
var currentTime = new GlideDateTime();
var enterMessage = gs.getMessage("Entering a p1 alert state. We predicted a p1 might occur within 20 minutes with a confidence of {0}%. It might be due to exhaustion in the default semaphore set and/or an unusually high number of transactions.", Math.round(confidence));
gs.warn(enterMessage);
var grEnter = new GlideRecord("diagnostic_event");
grEnter.initialize();
grEnter.setValue('name', "P1 Predicted");
grEnter.setValue('detail', enterMessage);
grEnter.setValue('severity', "Warning");
grEnter.setValue('status', "Open");
grEnter.setValue('reported_on', currentTime);
grEnter.insert();
var ongoingMessage = gs.getMessage("Ongoing P1 alert state.");
gs.info(ongoingMessage);
var grOngoing = new GlideRecord("diagnostic_event");
grOngoing.initialize();
grOngoing.setValue('name', "P1 Predicted");
grOngoing.setValue('severity', "Warning");
grOngoing.setValue('detail', ongoingMessage);
grOngoing.setValue('status', "Information");
grOngoing.setValue('reported_on', currentTime);
grOngoing.insert();
// Log P1 Prediction in GCF
var sm = new GCFSampleMap();
// GCFSampleMap can sometimes recognize currentTime.getValue as a Java Object rather than a string, needs casting
sm.put('start', String(currentTime.getValue()));
GCFCollector.recordUsageEvent('custom_metric', 'app_insights_usage', 'p1_predicted', sm);
} else
this._setState(state, true);
} else
this._setState(state, false);
grPredict.deleteRecord();
}
},
_getState: function(maxCoolDown) {
// Get most recent P1 Predict diagnostic event
var isPastCoolDown = false;
var gr = new GlideRecord('diagnostic_event');
gr.addQuery('name', "P1 Predicted");
gr.addQuery('status', "Information");
gr.orderByDesc('sys_created_on');
gr.query();
// If there is no information record, the state remains false
if (gr.next()) {
var lastAlert = new GlideDateTime(gr.getValue('reported_on'));
var cooldown = new GlideDateTime();
cooldown.addSeconds(-1 * maxCoolDown * 60);
isPastCoolDown = cooldown.before(lastAlert);
}
return isPastCoolDown;
},
_setState: function(state, resetState) {
var gr = new GlideRecord('diagnostic_event');
gr.addQuery('name', "P1 Predicted");
gr.addQuery('status', "Information");
gr.orderByDesc('sys_created_on');
gr.query();
if (gr.next()) {
if (!state) {
gr.setValue('detail', gs.getMessage("Ending current p1 alert state. We no longer predict a p1 might occur."));
gr.setValue('status', "Closed");
gr.setValue('reported_on', new GlideDateTime());
}
if (resetState)
gr.setValue('reported_on', new GlideDateTime());
gr.update();
}
},
type: 'P1Model'
};Sys ID
83d8e239c7602010ff7667617ec2604e