Name
global.DiscoveryPageManager
Description
No description available
Script
var DiscoveryPageManager;
(function() {
/**********************************************
Constants
***********************************************/
var FILE_NAME = 'DiscoveryPageManager',
LOG_PREFIX = '[' + FILE_NAME + ']';
/**********************************************
Local Variables
***********************************************/
var nodeLogger = new sn_automation.AutomationAPI().getNodeLogger();
/**********************************************
Public API
***********************************************/
DiscoveryPageManager = {
onPageReceived: _onPageReceived,
isPaginationFinished: _isPaginationFinished,
getPageCountRecordByOutputEccQueue: _getPageCountRecordByOutputEccQueue,
getPageDetails: _getPageDetails
};
function _onPageReceived(outputEccQueue, page, updatedBy, statusId) {
var outputEccLogPrefix = '[' + outputEccQueue + ']';
nodeLogger.info(LOG_PREFIX + outputEccLogPrefix + ' Page received. Updating pagination count tracking...');
nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' Inputs:\n'
+ 'Page: ' + JSON.stringify(page)) + '\n'
+ 'Updated By [Input ECC Queue]: ' + updatedBy;
// Optimization for not processing single paged executions
if (parseInt(page.number) == 1 && page.isLastPage == true)
return true;
// perform a lock to prevent double insertion
var key = "sn_discovery_page_count_" + outputEccQueue;
var mutex_metric_name = 'DiscoveryPageManager[' + outputEccQueue + ']';
var mutex = new SelfCleaningMutex(key, mutex_metric_name);
mutex.setSpinWait(gs.getProperty('glide.discovery.pagination.lock_spin_wait', 100)); // 100ms
mutex.setMaxSpins(gs.getProperty('glide.discovery.pagination.lock_max_spins', 600)); // 1min
// quit if a mutex can't be grabbed
if (!mutex.get()) {
var message = 'Unable to create to lock on ' + FILE_NAME + 'mutex for: ' + key;
nodeLogger.warn(LOG_PREFIX + outputEccLogPrefix + ' ' + message);
DiscoveryLogger.warn(message, FILE_NAME);
return;
}
try {
var pageCountGlideRecord = _updatePageCountTracking(outputEccQueue, page, updatedBy, statusId);
return _isPaginationFinished(outputEccQueue, updatedBy, pageCountGlideRecord);
} finally {
mutex.release();
}
}
function _isPaginationFinished(outputEccQueue, handledInputEccQueueId, pageCountGlideRecord) {
var outputEccLogPrefix = '[' + outputEccQueue + ']';
if (!pageCountGlideRecord)
pageCountGlideRecord = _getPageCountRecordByOutputEccQueue(outputEccQueue);
if (pageCountGlideRecord == null) {
nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' No page count tracking info found for ECC queue ' + outputEccQueue);
return true;
}
var pageReceived = pageCountGlideRecord.getValue('pages_received'),
totalPagesToProcess = pageCountGlideRecord.getValue('total_pages');
nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' Current state: \n'
+ 'Pages Received: ' + pageReceived + '\n'
+ 'Total Pages: ' + (totalPagesToProcess || 'Unknown') + '\n')
+ 'Input ECC Queue: ' + handledInputEccQueueId;
var areCountersEqual = JSUtil.notNil(totalPagesToProcess) && (pageReceived == totalPagesToProcess);
nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' Pagination '
+ (areCountersEqual ? 'is' : 'isn\'t')
+ ' finished');
/*
* 'last_updated_by' column was created in case multiple sensor threads finished processing pages and are about
* to update the 'Completed' count at the same time
* AFTER pages were processed by the sensor [which means that is some cases pageReceived = totalPagesToProcess], the threads are checking
* for pagination completion on 'DiscoverySensorJob'
* This causes the answer to be 'true' for these threads, which causes the 'Completed' count on the status to be higher then 'Started' count
* We will return 'true' not only when the counters are equal, but also when the input ECC queue we are currently handling is also
* the last one responsible for updating the page count record
* This should ensure that for every X pages [= input ECC queues], only 1 will return 'true' for this query
*/
var isPaginationFinished = areCountersEqual && ((pageCountGlideRecord.last_updated_by + '') == handledInputEccQueueId);
return isPaginationFinished;
}
function _getPageCountRecordByOutputEccQueue(outputEccQueue) {
var pageCountGlideRecord = new GlideRecord('sn_discovery_page_count');
pageCountGlideRecord.addQuery('output_ecc_queue', outputEccQueue);
pageCountGlideRecord.query();
if (pageCountGlideRecord.next())
return pageCountGlideRecord;
return null;
}
function _getPageDetails(probe) {
var pageNumber = probe.getParameter("orchestrator_page");
if (pageNumber == null)
pageNumber = '1';
var multiPageNumber = probe.getParameter("payload_page");
if (multiPageNumber != null)
pageNumber = pageNumber + '.' + (parseInt(multiPageNumber) + 1);
var isLastPaginatedPage = probe.getParameter("is_last_page") == null || probe.getParameter("is_last_page") == "true";
return {
number: pageNumber,
isLastPage: isLastPaginatedPage
};
}
/**********************************************
Private Helper Functions
***********************************************/
function _updatePageCountTracking(outputEccQueue, page, updatedBy, statusId) {
var outputEccLogPrefix = '[' + outputEccQueue + ']';
var pageCountGlideRecord = _getPageCountRecordByOutputEccQueue(outputEccQueue);
if (pageCountGlideRecord == null) {
var message = 'No page count tracking info found for ECC queue ' + outputEccQueue;
nodeLogger.info(LOG_PREFIX + outputEccLogPrefix + ' ' + message);
pageCountGlideRecord = new GlideRecord('sn_discovery_page_count');
pageCountGlideRecord.initialize();
pageCountGlideRecord.setValue('output_ecc_queue', outputEccQueue);
pageCountGlideRecord.setValue('discovery_status', statusId);
pageCountGlideRecord.insert();
}
var processedPages = pageCountGlideRecord.pages_received++;
pageCountGlideRecord.pages_received = processedPages;
pageCountGlideRecord.last_updated_by = updatedBy;
// If this is the last page, update the total pages needs to be process for this execution
// If isLastPage is 'true' or if it doesn't exist [not an orchestrator page!]
if (page && (!page.hasOwnProperty('isLastPage') || page.isLastPage))
pageCountGlideRecord.total_pages = parseInt(page.number) || 1;
if (pageCountGlideRecord.update()) {
nodeLogger.info(LOG_PREFIX + outputEccLogPrefix + ' Update finished successfully. Processed ' + processedPages + ' out of ' + (pageCountGlideRecord.total_pages || 'Unknown [Waiting for last page]'));
return pageCountGlideRecord;
}
var message = 'Error updating page counters.\nThis may affect the state of the discovery';
nodeLogger.error(LOG_PREFIX + outputEccLogPrefix + ' ' + message);
DiscoveryLogger.error(message, FILE_NAME);
}
})();
Sys ID
5b5dbf100f2700106dc4fe39b4767e17