Name

global.DiscoveryPageManager

Description

No description available

Script

var DiscoveryPageManager;

(function() {
  /**********************************************
  				  Constants
  ***********************************************/

  var FILE_NAME = 'DiscoveryPageManager',
  	LOG_PREFIX = '[' + FILE_NAME + ']';

  /**********************************************
  			    Local Variables
  ***********************************************/
  
  var nodeLogger = new sn_automation.AutomationAPI().getNodeLogger();
  
  /**********************************************
  				  Public API
  ***********************************************/
  DiscoveryPageManager = {
  	onPageReceived: _onPageReceived,
  	isPaginationFinished: _isPaginationFinished,
  	getPageCountRecordByOutputEccQueue: _getPageCountRecordByOutputEccQueue,
  	getPageDetails: _getPageDetails
  };
  
  function _onPageReceived(outputEccQueue, page, updatedBy, statusId) {
  	var outputEccLogPrefix = '[' + outputEccQueue + ']';
  	nodeLogger.info(LOG_PREFIX + outputEccLogPrefix + ' Page received. Updating pagination count tracking...');
  	nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' Inputs:\n' 
  					+ 'Page: ' + JSON.stringify(page)) + '\n'
  					+ 'Updated By [Input ECC Queue]: ' + updatedBy;
  	
  	// Optimization for not processing single paged executions
  	if (parseInt(page.number) == 1 && page.isLastPage == true)
  		return true;

  	// perform a lock to prevent double insertion
  	var key = "sn_discovery_page_count_" + outputEccQueue;
  	var mutex_metric_name = 'DiscoveryPageManager[' + outputEccQueue + ']';
  	var mutex = new SelfCleaningMutex(key, mutex_metric_name);
  	mutex.setSpinWait(gs.getProperty('glide.discovery.pagination.lock_spin_wait', 100)); // 100ms
  	mutex.setMaxSpins(gs.getProperty('glide.discovery.pagination.lock_max_spins', 600)); // 1min
  	
  	// quit if a mutex can't be grabbed
  	if (!mutex.get()) {
  		var message = 'Unable to create to lock on ' + FILE_NAME +  'mutex for: ' + key;
  		nodeLogger.warn(LOG_PREFIX + outputEccLogPrefix + ' ' + message);
  		DiscoveryLogger.warn(message, FILE_NAME);
  		return;
  	}

  	try {
  		var pageCountGlideRecord = _updatePageCountTracking(outputEccQueue, page, updatedBy, statusId);

  		return _isPaginationFinished(outputEccQueue, updatedBy, pageCountGlideRecord);
  	} finally {
  		mutex.release();
  	}
  }
  
  function _isPaginationFinished(outputEccQueue, handledInputEccQueueId, pageCountGlideRecord) {
  	var outputEccLogPrefix = '[' + outputEccQueue + ']';
  	
  	if (!pageCountGlideRecord)
  		pageCountGlideRecord = _getPageCountRecordByOutputEccQueue(outputEccQueue);
  	
  	if (pageCountGlideRecord == null) {
  		nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' No page count tracking info found for ECC queue ' + outputEccQueue);
  		return true;
  	}

  	var pageReceived = pageCountGlideRecord.getValue('pages_received'),
  		totalPagesToProcess = pageCountGlideRecord.getValue('total_pages');
  	
  	nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' Current state: \n'
  					+ 'Pages Received: ' + pageReceived + '\n'
  					+ 'Total Pages: ' + (totalPagesToProcess || 'Unknown') + '\n')
  					+ 'Input ECC Queue: ' + handledInputEccQueueId;
  	
  	var areCountersEqual = JSUtil.notNil(totalPagesToProcess) && (pageReceived == totalPagesToProcess);
  	nodeLogger.debug(LOG_PREFIX + outputEccLogPrefix + ' Pagination ' 
  					+ (areCountersEqual ? 'is' : 'isn\'t')
  					+ ' finished');
  	
  	/* 
  	 * 'last_updated_by' column was created in case multiple sensor threads finished processing pages and are about
  	 * to update the 'Completed' count at the same time
  	 * AFTER pages were processed by the sensor [which means that is some cases pageReceived = totalPagesToProcess], the threads are checking
  	 * for pagination completion on 'DiscoverySensorJob'
  	 * This causes the answer to be 'true' for these threads, which causes the 'Completed' count on the status to be higher then 'Started' count
  	 * We will return 'true' not only when the counters are equal, but also when the input ECC queue we are currently handling is also
  	 * the last one responsible for updating the page count record
  	 * This should ensure that for every X pages [= input ECC queues], only 1 will return 'true' for this query
  	 */
  	var isPaginationFinished = areCountersEqual && ((pageCountGlideRecord.last_updated_by + '') == handledInputEccQueueId);
  	
  	return isPaginationFinished;
  }
  
  function _getPageCountRecordByOutputEccQueue(outputEccQueue) {
  	var pageCountGlideRecord = new GlideRecord('sn_discovery_page_count');
  	pageCountGlideRecord.addQuery('output_ecc_queue', outputEccQueue);
  	pageCountGlideRecord.query();
  	
  	if (pageCountGlideRecord.next())
  		return pageCountGlideRecord;
  	
  	return null;
  }
  
  function _getPageDetails(probe) {
  	var pageNumber = probe.getParameter("orchestrator_page");

  	if (pageNumber == null)
  		pageNumber = '1';
  	var multiPageNumber = probe.getParameter("payload_page");

  	if (multiPageNumber != null)
  		pageNumber = pageNumber + '.' + (parseInt(multiPageNumber) + 1);	
  	var isLastPaginatedPage = probe.getParameter("is_last_page") == null || probe.getParameter("is_last_page") == "true";
  	
  	return {
  		number: pageNumber,
  		isLastPage: isLastPaginatedPage
  	};
  }

  /**********************************************
  			Private Helper Functions
  ***********************************************/
  
  function _updatePageCountTracking(outputEccQueue, page, updatedBy, statusId) {
  	var outputEccLogPrefix = '[' + outputEccQueue + ']';
  	var pageCountGlideRecord = _getPageCountRecordByOutputEccQueue(outputEccQueue);

  	if (pageCountGlideRecord == null) {
  		var message = 'No page count tracking info found for ECC queue ' + outputEccQueue;
  		nodeLogger.info(LOG_PREFIX + outputEccLogPrefix + ' ' + message);
  		
  		pageCountGlideRecord = new GlideRecord('sn_discovery_page_count');
  		pageCountGlideRecord.initialize();
  		pageCountGlideRecord.setValue('output_ecc_queue', outputEccQueue);
  		pageCountGlideRecord.setValue('discovery_status', statusId);

  		pageCountGlideRecord.insert();
  	}

  	var processedPages = pageCountGlideRecord.pages_received++;
  	pageCountGlideRecord.pages_received = processedPages;
  	pageCountGlideRecord.last_updated_by = updatedBy;

  	// If this is the last page, update the total pages needs to be process for this execution
  	// If isLastPage is 'true' or if it doesn't exist [not an orchestrator page!]
  	if (page && (!page.hasOwnProperty('isLastPage') || page.isLastPage))
  		pageCountGlideRecord.total_pages = parseInt(page.number) || 1;

  	if (pageCountGlideRecord.update()) {
  		nodeLogger.info(LOG_PREFIX + outputEccLogPrefix + ' Update finished successfully. Processed ' + processedPages + ' out of ' + (pageCountGlideRecord.total_pages || 'Unknown [Waiting for last page]'));
  		
  		return pageCountGlideRecord;
  	}
  		
  	var message = 'Error updating page counters.\nThis may affect the state of the discovery';
  	nodeLogger.error(LOG_PREFIX + outputEccLogPrefix + ' ' + message);
  	DiscoveryLogger.error(message, FILE_NAME);
  }
})();

Sys ID

5b5dbf100f2700106dc4fe39b4767e17

Offical Documentation

Official Docs: