Name
sn_nlu_workbench.NLUExpertFeedbackUtil
Description
Utilities to get the Data Labeling (Expert Feedback Loop) information.
Script
var NLUExpertFeedbackUtil = Class.create();
(function() {
var tables = sn_nlu_workbench.NLUWorkbenchConstants.tables;
var coreTables = global.NLUConstants.tables;
var TODO_FURTHER_REVIEW_FIELDS = {
LABEL: 'lct_suggested_label',
UTTERANCE_REF: 'lft_utterance_reference'
};
var tabNames = {
TODO: 'todo',
DONE: 'done',
FURTHER_REVIEW: 'further_review'
};
var outcomes = {
MATCH: 'positive',
MISMATCH: 'negative',
UNSURE: 'notsure',
IRRELEVANT: 'irrelevant'
};
NLUExpertFeedbackUtil.getModelOptimizeInfo = function(modelId) {
var modelStatusGr = global.NLUModel.getModelStatusGr(modelId);
var trainedVersion = modelStatusGr.getValue('trained_version');
var latestTrainOptimizeInfo = NLUBatchTestExecution.getReportInfo(modelId, true, trainedVersion, true) || {};
var latestModelPublished = trainedVersion === modelStatusGr.getValue('published_version');
return {
executionId: latestTrainOptimizeInfo.executionId,
definitionId: latestTrainOptimizeInfo.definitionId,
status: latestTrainOptimizeInfo.status,
lastTrainedVersionOptimizedOn: latestTrainOptimizeInfo.lastTrainedVersionOptimizedOn,
latestModelPublished: latestModelPublished
};
};
NLUExpertFeedbackUtil.labeledCountSince = function(modelName, fromDate) {
var ga = new GlideAggregate(tables.ULT);
ga.addQuery('product', 'nlu');
if (fromDate) ga.addQuery('sys_updated_on', '>=', new GlideDateTime(fromDate));
ga.addEncodedQuery('labelSTARTSWITHintent:' + modelName + '.^ORlabelSTARTSWITHmodel:' + modelName);
ga.addAggregate('COUNT');
ga.query();
return ga.next() && parseInt(ga.getAggregate('COUNT')) || 0;
};
NLUExpertFeedbackUtil.getLabelFromReference = function(label_reference) {
var gr = new GlideRecord(coreTables.SYS_NLU_INTENT);
if (gr.get(label_reference))
return 'intent:' + gr.model.name + '.' + gr.name;
return '';
};
NLUExpertFeedbackUtil.getReferenceFromLabel = function(label) {
if (!label || !label.includes(":")) return null;
var labelTable = '';
var query = '';
var labelArr1 = label.split(":");
if (labelArr1[0] === 'intent' && labelArr1[1].includes(".")) {
labelTable = coreTables.SYS_NLU_INTENT;
var labelArr2 = labelArr1[1].split(".");
query = 'model.name=' + labelArr2[0] + '^name=' + labelArr2[1];
} else if (labelArr1[0] === 'model') {
labelTable = coreTables.SYS_NLU_MODEL;
query = 'name=' + labelArr1[1];
}
if (!query) return null;
var gr = new GlideRecord(labelTable);
gr.addEncodedQuery(query);
gr.setLimit(1);
gr.orderBy('sys_updated_on');
gr.query();
if (gr.next()) {
return {
label_table: labelTable,
label_reference: gr.getUniqueValue()
};
}
return null;
};
NLUExpertFeedbackUtil.prototype = {
initialize: function() {
this.modelToIntentMap = {};
this.lctDeletes = [];
this.ultDeletes = [];
this.uftDeletes = [];
this.uncategorizedUtterances = {};
this.uncategorizedUtterances[tabNames.TODO] = 0;
this.uncategorizedUtterances[tabNames.DONE] = 0;
},
getModelList: function() {
this._populateModelIntentMap(tables.LCT, 'suggested_label');
this._populateModelIntentMap(tables.ULT, 'label');
var modelList = [];
for (var modelName in this.modelToIntentMap) {
var aggr = new GlideAggregate(coreTables.SYS_NLU_INTENT);
aggr.addQuery('model.name', modelName);
aggr.addQuery('name', this.modelToIntentMap[modelName]);
aggr.groupBy('model');
aggr.addAggregate('COUNT');
aggr.query();
if (aggr.next()) {
var statusGr = new GlideRecord(coreTables.SYS_NLU_MODEL_STATUS);
statusGr.addQuery('model.name', modelName);
statusGr.query();
modelList.push({
modelId: aggr.getValue('model'),
displayName: aggr.getDisplayValue('model'),
intentCount: aggr.getAggregate('COUNT') || 0,
name: modelName,
language: aggr.model.language,
lastTunedOn: statusGr.next() ? statusGr.getValue('last_tuned_on') : null
});
}
}
var modelAgg = new GlideAggregate(coreTables.SYS_NLU_MODEL_STATUS);
modelAgg.addQuery('published_version', '>=', '1');
modelAgg.addAggregate('COUNT');
modelAgg.query();
return {
hasPublishedModels: modelAgg.next() && modelAgg.getAggregate('COUNT') > 0,
modelList: modelList,
uncategorizedUtterances: this.uncategorizedUtterances
};
},
getIntentList: function(modelId) {
var intentsData = [];
var intentGr = new GlideRecord(coreTables.SYS_NLU_INTENT);
intentGr.addQuery('model', modelId);
intentGr.query();
var modelName = null;
var intentDataMap = {};
while (intentGr.next()) {
if (!modelName) modelName = intentGr.model.name;
var label = 'intent:' + modelName + '.' + intentGr.name;
intentDataMap[label] = {
intentId: intentGr.getUniqueValue(),
name: intentGr.getValue('name'),
label: label
};
}
var reviewProgressMap = this.getReviewProgressMap(Object.keys(intentDataMap), modelId);
for (var _label in reviewProgressMap) {
intentDataMap[_label].reviewProgress = reviewProgressMap[_label];
intentsData.push(intentDataMap[_label]);
}
var optimizeData = NLUExpertFeedbackUtil.getModelOptimizeInfo(modelId);
return {
optimizeData: optimizeData,
labeledCountSinceLastOptimized: NLUExpertFeedbackUtil.labeledCountSince(modelName, optimizeData.lastTrainedVersionOptimizedOn),
intents: intentsData
};
},
getReviewProgressMap: function(labelList, modelId) {
var reviewProgressMap = {};
var doneData = this._getUtteranceCountForLabels(
tables.ULT,
'label',
labelList,
modelId
);
var todoFurtherReviewData = this._getTodoFurtherReviewCountForLabels(
labelList
);
var todoData = todoFurtherReviewData[tabNames.TODO] || {};
var furtherReviewData = todoFurtherReviewData[tabNames.FURTHER_REVIEW] || {};
labelList.forEach(function(_label) {
var todo = todoData[_label] || 0;
var done = doneData[_label] || 0;
var furtherReview = furtherReviewData[_label] || 0;
if (!_label) { // For uncategorised utterances:
todo += furtherReview;
furtherReview = 0;
}
if (todo + done + furtherReview > 0) {
reviewProgressMap[_label] = {};
reviewProgressMap[_label][tabNames.TODO] = todo;
reviewProgressMap[_label][tabNames.DONE] = done;
reviewProgressMap[_label][tabNames.FURTHER_REVIEW] = furtherReview;
}
});
return reviewProgressMap;
},
/**
*
* feedbackConfig example
* {
* "todo": {
* "sysId1": {
* "feedback": "positive"
* }
* },
* "done": {
* "sysId11": {
* "feedback": "negative",
* "correct_label_reference": "sysId3434"
* }
* },
* "review": {
* "sysId23": {
* "feedback": "notsure"
* }
* }
* }
*/
updateFeedback: function(selectedIntent, feedbackConfig) {
try {
if (Object.keys(feedbackConfig[tabNames.TODO]).length > 0) {
this._updateTodoFeedback(feedbackConfig[tabNames.TODO]);
}
if (Object.keys(feedbackConfig[tabNames.DONE]).length > 0) {
this._updateDoneFeedback(feedbackConfig[tabNames.DONE]);
}
if (Object.keys(feedbackConfig[tabNames.FURTHER_REVIEW]).length > 0) {
this._updateReviewFeedback(feedbackConfig[tabNames.FURTHER_REVIEW]);
}
var modelIntentStr = (selectedIntent && selectedIntent.substring(7)) || false;
var tgtModel = modelIntentStr && modelIntentStr.split('.')[0];
var modelGr = global.NLUModel.getGRByName(tgtModel);
var map = this.getReviewProgressMap([selectedIntent], modelGr ? modelGr.getValue('sys_id') : null);
if (map && map[selectedIntent]) {
return map[selectedIntent];
}
} catch (error) {
throw error;
}
return {};
},
_updateTodoFeedback: function(todoConfig) {
var sysIds = Object.keys(todoConfig);
for (var i = 0; i < sysIds.length; i++) {
var sysId = sysIds[i];
var correct_label_reference = todoConfig[sysId].correct_label_reference || '';
var feedback = todoConfig[sysId].feedback;
var lctData = this._getLCTRecord(sysId);
if (feedback === outcomes.MATCH) {
// 1. Create record in ULT (label_type: positive)
this._createULTRecord({
text: lctData.text,
product: lctData.product,
source: lctData.source,
label: lctData.label,
label_type: outcomes.MATCH
});
// 2. Remove record in LCT
this._markLCTDelete(sysId);
} else if (feedback === outcomes.MISMATCH || feedback === outcomes.IRRELEVANT) {
// 1. Create record in ULT (label_type: negative, correct_label_reference if given)
this._createULTRecord({
text: lctData.text,
product: lctData.product,
source: lctData.source,
label: lctData.label,
label_type: feedback,
correct_label_reference: correct_label_reference
});
// 2. Remove record in LCT
this._markLCTDelete(sysId);
} else if (feedback === outcomes.UNSURE) {
// 1. Create record in UFT (label_type: unsure)
this._createUFTRecord({
utterance_reference: sysId,
utterance_table: tables.LCT,
label_type: outcomes.UNSURE
});
}
}
this._deleteRecords(tables.LCT);
},
_updateDoneFeedback: function(doneConfig) {
var sysIds = Object.keys(doneConfig);
for (var i = 0; i < sysIds.length; i++) {
var sysId = sysIds[i];
var feedback = doneConfig[sysId].feedback;
var correct_label_reference = doneConfig[sysId].correct_label_reference || '';
var ultRecord = new GlideRecord(tables.ULT);
ultRecord.addQuery('sys_id', sysId);
ultRecord.query();
if (feedback === outcomes.MATCH) {
if (ultRecord.next()) {
// Update ULT record label_type
ultRecord.setValue('label_type', feedback);
ultRecord.setValue('label_table', coreTables.SYS_NLU_INTENT);
ultRecord.setValue('correct_label_reference', '');
ultRecord.update();
return ultRecord.getUniqueValue();
}
} else if (feedback === outcomes.MISMATCH || feedback === outcomes.IRRELEVANT) {
if (ultRecord.next()) {
ultRecord.setValue('label_type', feedback);
if (correct_label_reference) {
ultRecord.setValue('correct_label_reference', correct_label_reference);
if (gs.nil(ultRecord.getValue('label_table')))
ultRecord.setValue('label_table', coreTables.SYS_NLU_INTENT);
} else {
ultRecord.setValue('correct_label_reference', '');
}
ultRecord.update();
}
} else if (feedback === outcomes.UNSURE) {
// Create record in LCT - take reference
var ultData = this._getULTRecord(sysId);
var lctSysId = global.MLLabelCandidate.createRecord({
text: ultData.text,
suggested_label: ultData.label,
source: ultData.source,
product: ultData.product
});
// Create UFT record with label_type - unsure (notsure) with utterance_reference as the above LCT record sys_id
this._createUFTRecord({
utterance_reference: lctSysId,
utterance_table: tables.LCT,
label_type: feedback
});
// delete ULT record
this._markULTDelete(sysId);
}
}
this._deleteRecords(tables.ULT);
},
_updateReviewFeedback: function(furtherReviewConfig) {
var sysIds = Object.keys(furtherReviewConfig);
for (var i = 0; i < sysIds.length; i++) {
var sysId = sysIds[i];
var feedback = furtherReviewConfig[sysId].feedback;
var correct_label_reference = furtherReviewConfig[sysId].correct_label_reference || '';
var uftData = this._getUFTRecord(sysId);
var lctData = this._getLCTRecord(uftData.utterance_reference);
var ultSysId;
// 1. Create record in ULT - take note of sysId
if (feedback === outcomes.MATCH) {
ultSysId = this._createULTRecord({
text: lctData.text,
product: lctData.product,
source: lctData.source,
label: lctData.label,
label_type: outcomes.MATCH,
correct_label_reference: ''
});
} else if (feedback === outcomes.MISMATCH || feedback === outcomes.IRRELEVANT) {
ultSysId = this._createULTRecord({
text: lctData.text,
product: lctData.product,
source: lctData.source,
label: lctData.label,
label_type: feedback,
correct_label_reference: feedback === outcomes.IRRELEVANT ? '' : correct_label_reference
});
}
// 2. Update record of UFT (utterance_reference to sysId from point 1.)
this._updateUtteranceReference({
sys_id: sysId,
utterance_reference: ultSysId,
utterance_table: tables.ULT,
label_type: feedback,
correct_label_reference: correct_label_reference
});
// 3. Remove record from LCT
this._markLCTDelete(uftData.utterance_reference);
}
this._deleteRecords(tables.LCT);
},
_updateUtteranceReference: function(data) {
var uftRecord = new GlideRecord(tables.LABEL_USER_FEEDBACK);
uftRecord.addQuery('sys_id', data.sys_id);
uftRecord.query();
if (uftRecord.next()) {
uftRecord.setValue('utterance_reference', data.utterance_reference);
uftRecord.setValue('utterance_table', data.utterance_table);
uftRecord.setValue('label_type', data.label_type);
uftRecord.setValue('correct_label_reference', data.correct_label_reference);
uftRecord.setValue('utterance_table', tables.ULT);
uftRecord.update();
}
},
_getLCTRecord: function(sysId) {
var gr = new GlideRecord(tables.LCT);
gr.addQuery('sys_id', sysId);
gr.query();
if (gr.next()) {
return {
text: gr.getValue('text'),
product: gr.getValue('product'),
source: gr.getValue('source'),
label: gr.getValue('suggested_label')
};
}
return null;
},
_getULTRecord: function(sysId) {
var gr = new GlideRecord(tables.ULT);
gr.addQuery('sys_id', sysId);
gr.query();
if (gr.next()) {
return {
text: gr.getValue('text'),
label: gr.getValue('label'),
label_type: gr.getValue('label_type'),
correct_label_reference: gr.getValue('correct_label_reference'),
product: gr.getValue('product'),
source: gr.getValue('source')
};
}
return null;
},
_getUFTRecord: function(sysId) {
var gr = new GlideRecord(tables.LABEL_USER_FEEDBACK);
gr.addQuery('sys_id', sysId);
gr.query();
if (gr.next()) {
return {
utterance_reference: gr.getValue('utterance_reference'),
utterance_table: gr.getValue('utterance_table'),
label_type: gr.getValue('label_type'),
correct_label_reference: gr.getValue('correct_label_reference'),
user: gr.getValue('user')
};
}
return null;
},
/**
* To create a record in ml_labeled_data table (ULT)
*/
_createULTRecord: function(data) {
var gr = new GlideRecord(tables.ULT);
gr.initialize();
gr.setValue('text', data.text);
gr.setValue('label_type', data.label_type);
gr.setValue('source', data.source);
gr.setValue('product', data.product);
gr.setValue('label', data.label);
gr.setValue('usage', 'nlu_model_train');
var labelRef = NLUExpertFeedbackUtil.getReferenceFromLabel(data.label);
if (labelRef) {
gr.setValue('label_reference', labelRef.label_reference);
gr.setValue('label_table', labelRef.label_table);
}
if (data.correct_label_reference) {
gr.setValue('correct_label_reference', data.correct_label_reference);
gr.setValue('correct_label', NLUExpertFeedbackUtil.getLabelFromReference(data.correct_label_reference));
gr.setValue('label_table', coreTables.SYS_NLU_INTENT);
}
gr.insert();
return gr.getUniqueValue();
},
/**
* To create a record in ml_label_candidate table (LCT)
*/
_createLCTRecord: function() {},
/**
* To create a record in ml_label_user_feedback table (UFT)
*/
_createUFTRecord: function(data) {
var gr = new GlideRecord(tables.LABEL_USER_FEEDBACK);
gr.initialize();
gr.setValue('utterance_reference', data.utterance_reference);
gr.setValue('utterance_table', data.utterance_table);
gr.setValue('label_type', data.label_type);
gr.insert();
return gr.getUniqueValue();
},
_markLCTDelete: function(sysId) {
this.lctDeletes.push(sysId);
},
_markULTDelete: function(sysId) {
this.ultDeletes.push(sysId);
},
_markUFTDelete: function(sysId) {
this.uftDeletes.push(sysId);
},
_deleteRecords: function(table) {
var records = [];
if (table === tables.LCT) {
records = this.lctDeletes;
global.MLLabelCandidate.deleteRecords('sys_idIN' + records.join(','));
this.lctDeletes = [];
} else if (table === tables.ULT) {
records = this.ultDeletes;
global.MLLabeledData.deleteRecords('sys_idIN' + records.join(','));
this.ultDeletes = [];
}
},
_getLabelListQuery: function(field, labels) {
var query = field + 'IN' + labels.join(',');
var index = labels.indexOf('');
if (index > -1) {
query = field + 'ISEMPTY';
var other = labels.filter(function(x) {
return x != '';
});
if (other.length > 0)
query += '^OR' + field + 'IN' + other.join(',');
}
return query;
},
_getUtteranceCountForLabels: function(tableName, field, labels, modelId) {
var aggr = new GlideAggregate(tableName);
aggr.addQuery('product', 'nlu');
aggr.addQuery('source', 'virtual_agent');
aggr.addEncodedQuery(this._getLabelListQuery(field, labels));
aggr.addAggregate('COUNT', field);
var statusGr = new GlideRecord('sys_nlu_model_status');
statusGr.addQuery('model', modelId);
statusGr.query();
if (statusGr.next() && statusGr.getValue('last_tuned_on')) {
aggr.addQuery('sys_updated_on', '>=', statusGr.getValue('last_tuned_on'));
}
aggr.query();
var labelReviewCountMap = {};
while (aggr.next()) {
var label = aggr.getValue(field);
labelReviewCountMap[label] =
parseInt(aggr.getAggregate('COUNT', field)) || 0;
}
return labelReviewCountMap;
},
_getTodoFurtherReviewCountForLabels: function(labels) {
var result = {};
var todo = {};
var furtherReview = {};
var aggr = new GlideAggregate(tables.DBVIEW_TODO_FURTHER_REVIEW);
aggr.groupBy(TODO_FURTHER_REVIEW_FIELDS.LABEL);
aggr.groupBy(TODO_FURTHER_REVIEW_FIELDS.UTTERANCE_REF);
aggr.addEncodedQuery(this._getLabelListQuery(TODO_FURTHER_REVIEW_FIELDS.LABEL, labels));
aggr.addAggregate('COUNT');
aggr.query();
while (aggr.next()) {
var label = aggr.getValue(TODO_FURTHER_REVIEW_FIELDS.LABEL);
var isFurtherReview = aggr.getValue(
TODO_FURTHER_REVIEW_FIELDS.UTTERANCE_REF
);
if (isFurtherReview)
furtherReview[label] =
(furtherReview[label] || 0) +
(parseInt(aggr.getAggregate('COUNT')) || 0);
else todo[label] = parseInt(aggr.getAggregate('COUNT')) || 0;
}
result[tabNames.TODO] = todo;
result[tabNames.FURTHER_REVIEW] = furtherReview;
return result;
},
_populateModelIntentMap: function(tableName, field) {
var aggr = new GlideAggregate(tableName);
aggr.addQuery('product', 'nlu');
aggr.addQuery('source', 'virtual_agent');
aggr.addAggregate('COUNT', field);
aggr.query();
while (aggr.next()) {
var label = aggr.getValue(field);
if (!label) {
var key = tableName === tables.LCT ? tabNames.TODO : tabNames.DONE;
this.uncategorizedUtterances[key] = aggr.getAggregate('COUNT', field) || 0;
} else if (label.indexOf('intent:') === 0) {
var modelIntentStr = label.substring('intent:'.length);
var model = modelIntentStr.split('.')[0];
var intent = modelIntentStr.substring(model.length + 1);
if (!this.modelToIntentMap.hasOwnProperty(model))
this.modelToIntentMap[model] = [];
if (this.modelToIntentMap[model].indexOf(intent) === -1)
this.modelToIntentMap[model].push(intent);
}
}
},
type: 'NLUExpertFeedbackUtil'
};
})();Sys ID
8c1168780723301028ef0a701ad3002c