Name

sn_itom_pattern.ContainerImageScanJob

Description

This script handles following 1. Create a entry for batch of images in sn_itom_pattern_container_image_scan_status 2. Update scan_stauts to error when discovery status is completed/cancelled and scan_status is in in_progress 3. Trigger a Scan Container Image pattern for non processed images

Script

var ContainerImageScanJob = Class.create();
ContainerImageScanJob.prototype = {
  _batchSize: parseInt(gs.getProperty('sn_itom_pattern.container_image_batch_size', '10')),
  _scan_status_table: 'sn_itom_pattern_container_image_scan_status',
  _rel_ci_table: 'cmdb_rel_ci',
  _sa_hash_table: 'sa_hash',
  _discovery_status_table: 'discovery_status',
  _oslv_image_table: 'cmdb_ci_oslv_image',
  _imageRepositryEntryMap: {}, //Map key is container image sys_id and value is (sydId- container Image entry Sys_id and version is container Image entry name)
  _repositryEntryMap: {}, //Map key is container repositry entry sys_id and value is container repositry name
  _imageIdsMap: {}, //Map key is container image sys Id and value is container image id
  _localImageTagMap: {}, //Map key is container image Id and value is cmdb_ci_docker_image_tag name
  _imageIdsList: [], // Array of Contianer Image Ids 


  initialize: function() {
      this.logger = new Logger('ContainerImageScanJob');
      this.LAST_CONTAINER_IMAGE_COPIED_TIME = 'last_container_image_copied_time';
  },
  run: function() {
      var scanStatusSysIds = [];
      var imageSysIds = [];

      this.logger.debug(' container image batch size is ' + this._batchSize);
      //Copy next batch of container images into Container Image Scan Status table  
      this._copyNextBatchImages();
  	
      // Check if there is a MID for image scan
      var midName = this._findImageScanCapabilityMid();
      if (!midName) {
          this.logger.debug('can not find MID Server with Scan Container Images capability');
          return;
      }

      // If there are previous images in status in_progress, then don't start new one
      var inProgress = this._countInProgressImages();
      if (inProgress > 0) {
          this.logger.debug('There are already ' + inProgress + ' images in in_progress state');
          return;
      }

      // Get the images not yet processed
      var imageScanStatusGr = new GlideRecord(this._scan_status_table);
      imageScanStatusGr.addQuery('scan_status', 'IN', ['', 'none']);
      imageScanStatusGr.setLimit(this._batchSize);
      imageScanStatusGr.orderBy('sys_created_on');
      imageScanStatusGr.query();

      if (!imageScanStatusGr.hasNext()) {
          this.logger.debug(' No images to scan');
          return;
      }
      
      while (imageScanStatusGr.next()) {
          var imageSysId = imageScanStatusGr.image + '';
          scanStatusSysIds.push(imageScanStatusGr.getUniqueValue());
          imageSysIds.push(imageSysId);
          this._imageIdsMap[imageSysId] = imageScanStatusGr.image.image_id + '';
          this._imageIdsList.push(imageScanStatusGr.image.image_id + '');
      }
      this.logger.debug(' Scan image_scan_stauts SysIds = ' + scanStatusSysIds);
      // Create discovery status
      var discoveryStatusId = this._createDiscoveryStatus();
      if (!discoveryStatusId) {
          this.logger.debug(' Failed to create discovery status');
          return;
      }
      this.logger.debug(' Add Scan jobs to discovery status ' + discoveryStatusId);
      // Get repositry and tag from Container repositry entry table
      this._getImagesRepoTag(imageSysIds);
      //Get repo tag for Local docker images 
      this._getLocalImageTag();
      //Pattern triggering logic
      this._triggerPattern(scanStatusSysIds, midName, discoveryStatusId);
  },

  /* Get repositry and tag from Container repositry entry table
     repositry name from cmdb_ci_container_repository table
     Version from cmdb_ci_container_repository_entry table
  */
  _getImagesRepoTag: function(imageSysIds) {
      var repositoryEntrySysIds = [];
      var relGr = new GlideRecord(this._rel_ci_table);
      relGr.addQuery("parent", "IN", imageSysIds);
      relGr.addQuery("type", "72e003db0b032200639be0d425673aa1"); // Provisioned From::Provisioned
      relGr.addQuery("child.sys_class_name", "cmdb_ci_container_repository_entry");
      relGr.query();
      while (relGr.next()) {
          var repositoryEntrySysId = relGr.child.sys_id + '';
          this._imageRepositryEntryMap[relGr.parent + ''] = {
              "sysId": repositoryEntrySysId,
              "version": relGr.child.name + ''
          };
          repositoryEntrySysIds.push(repositoryEntrySysId);
      }
      this.logger.debug('Repo Tag using Repositry entry Sys Ids =' + repositoryEntrySysIds);
      relGr = new GlideRecord(this._rel_ci_table);
      relGr.addQuery("parent", "IN", repositoryEntrySysIds);
      relGr.addQuery("type", "5f985e0ec0a8010e00a9714f2a172815"); // Hosted on::Hosts
      relGr.addQuery("child.sys_class_name", "cmdb_ci_container_repository");
      relGr.query();
      while (relGr.next()) {
          this._repositryEntryMap[relGr.parent + ''] = relGr.child.name + '';
          this.logger.debug(' For Repositry entry Sys ID ' + relGr.parent + ' repositroy Name is ' + relGr.child.name);
      }

  },
  
  /*Get repo tag for Local docker images
    Some images does not contain entry in cmdb_ci_container_repository_entry table.
    For those images get Docker image repo_tag from cmdb_ci_docker_image_tag table if exists
    cmdb_ci_docker_image_tag - name contains image repositry and version
  */
  _getLocalImageTag: function() {
      this.logger.debug(' Get Local docker Image tags for image ids= ' + this._imageIdsList);
      var imageTagGr = new GlideRecord("cmdb_ci_docker_image_tag");
      imageTagGr.addQuery("image_id", "IN", this._imageIdsList);
      imageTagGr.query();
      while (imageTagGr.next()) {
          var imageId = imageTagGr.image_id + '';
          var imageTag = imageTagGr.name + '';
          this._localImageTagMap[imageId] = imageTag;
          this.logger.debug(' For image Id ' + imageId + '  Image tag is ' + imageTag);

      }
  },
   
  // Logic to trigger  Scan Container Image pattern per Image
  _triggerPattern: function(scanStatusSysIds, midName, discoveryStatusId) {
      this.logger.debug(' Start construct ecc Queue payload for image_scan_status sysIds =' + scanStatusSysIds);
      var imageScanStatusGr = new GlideRecord(this._scan_status_table);
      imageScanStatusGr.addQuery("sys_id", "IN", scanStatusSysIds);
      imageScanStatusGr.query();
      while (imageScanStatusGr.next()) {
          var docker_image_sysid = imageScanStatusGr.image + '';
          if (!docker_image_sysid) {
              this.logger.debug('Container image reference id is not present for Image scan status sys id ' + imageScanStatusGr.sys_id);
              imageScanStatusGr.setValue('message', 'Container image reference id is not present');
              imageScanStatusGr.setValue('scan_status', 'skipped');
              imageScanStatusGr.update();
              continue;
          }
          var container_image_repo_tag = this._getRepoTagFromContainerEntry(imageScanStatusGr, docker_image_sysid);
          if (!container_image_repo_tag) {
              container_image_repo_tag = this._getRepoTagFromLocalImage(imageScanStatusGr, docker_image_sysid);
          }
  		// If container_image_repo_tag not contians, then no need to trigger pattern
          if (!container_image_repo_tag) {
              this.logger.debug(' Contianer Image Tag is not found for Scan Status ' + imageScanStatusGr.sys_id + ' , container image = ' + imageScanStatusGr.image.name);
              imageScanStatusGr.setValue('message', 'Repositry name/version not found in either tables(container repositry entry, docker image tag) for this image');
              imageScanStatusGr.setValue('scan_status', 'skipped');
              imageScanStatusGr.update();
              continue;
          }
          this.logger.debug(' Pattern triggering for Image scan status sys id ' + imageScanStatusGr.sys_id + ' ,container image = ' + imageScanStatusGr.image.name + ' , container image repo_tag=' + container_image_repo_tag);
          var eccQueue_id = this._sendPattern(docker_image_sysid, container_image_repo_tag, midName, discoveryStatusId);
          if (eccQueue_id) {
              // Update the scan status to in_progress
              imageScanStatusGr.setValue('scan_status', 'in_progress');
              imageScanStatusGr.setValue('discovery_status', discoveryStatusId);
              imageScanStatusGr.update();
              this.logger.debug(' Pattern triggered for Image scan status sys id ' + imageScanStatusGr.sys_id + ' ,container image repo_tag =' + container_image_repo_tag);
          } else {
              this.logger.debug(' EccQueue payload insertion failed for Image scan status sys id ' + imageScanStatusGr.sys_id + ' ,container image repo_tag=' + container_image_repo_tag);
          }
      }
  },

  // Get repo Tag from Container entry
  _getRepoTagFromContainerEntry: function(imageScanStatusGr, docker_image_sysid) {
      var repositryEntry = this._imageRepositryEntryMap[docker_image_sysid];
      if (typeof(repositryEntry) == 'undefined') {
          this.logger.debug('Container entry repo_tag is not present for container docker image = ' + imageScanStatusGr.image.name + ' , image scan status id = ' + imageScanStatusGr.sys_id);
          return;
      }
      var repoName = repoName = this._repositryEntryMap[repositryEntry['sysId']];
      if (typeof(repoName) == 'undefined') {
          this.logger.debug('Container repositry is not present for container docker image = ' + imageScanStatusGr.image.name + ' , image scan status id = ' + imageScanStatusGr.sys_id + ' , Container repositry entry sys_id ' + repositryEntry['sysId']);
          return;
      }
      var repo_tag = repoName + ':' + repositryEntry['version'];
      this.logger.debug('Getting Container image ' + imageScanStatusGr.image.name + ' repo tag from Repositry entry. repo_tag is ' + repo_tag);
      return repo_tag;
  },
  // Get repo Tag from cmdb_ci_docker_image_tag Map
  _getRepoTagFromLocalImage: function(imageScanStatusGr, docker_image_sysid) {
      var imageId = this._imageIdsMap[docker_image_sysid];
      if (typeof(imageId) == 'undefined') {
          this.logger.debug('image Id is not present for container docker image = ' + imageScanStatusGr.image.name + ' , image scan status id = ' + imageScanStatusGr.sys_id);
          return;
      }
      var localImageTag = this._localImageTagMap[imageId];
      if (typeof(localImageTag) == 'undefined') {
          this.logger.debug('Local Image Tag is not present for container docker image = ' + imageScanStatusGr.image.name + ' , image scan status id = ' + imageScanStatusGr.sys_id + ' , Image Id =' + imageId);
          return;
      }
      this.logger.debug('Getting Container image ' + imageScanStatusGr.image.name + ' repo tag from Docker Image Tag. repo_tag is ' + localImageTag);
      return localImageTag;
  },
  
  // Insert Ecc Queue pattern payload
  _sendPattern: function(docker_image_sysid, docker_image_name, midName, discoveryStatusId) {

      var payload = this._createEccPayload(docker_image_sysid, docker_image_name);
      // Send ECC message
      this.logger.debug('Inserting EccQueue payload for ' + docker_image_name);
      var eccQueueGr = new GlideRecord('ecc_queue');
      eccQueueGr.initialize();
      eccQueueGr.setValue('agent_correlator', discoveryStatusId);
      eccQueueGr.setValue('agent', 'mid.server.' + midName);
      eccQueueGr.setValue('source', docker_image_name);
      eccQueueGr.setValue('name', docker_image_name);
      eccQueueGr.setValue('payload', payload.toString());
      eccQueueGr.setValue('topic', 'HorizontalDiscoveryProbe');
      eccQueueGr.setValue('queue', 'output');
      eccQueueGr.setValue('state', 'ready');
      eccQueueGr.setValue('priority', '2');
      var insert_eccQueue_id = eccQueueGr.insert();
      return insert_eccQueue_id ? insert_eccQueue_id : null;

  },

  // Create Ecc Queue paylaod
  _createEccPayload: function(docker_image_sysid, docker_image_name) {
      this.logger.debug('Creating EccQueue payload for ' + docker_image_name);
      var payload = new XMLDocument2();
      payload.createElement("parameters");
      this._addParameter(payload, 'pattern_type', 'cmdb_ci_docker_image');
      this._addParameter(payload, 'patternId', '66cff07153f99110e3baddeeff7b123f');
      this._addParameter(payload, 'pattern', 'Scan Container Image');
      this._addParameter(payload, 'dscheduler', 'Hostless');
      this._addParameter(payload, 'probe', '4f64c6389f230200fe2ab0aec32e7068');
      this._addParameter(payload, 'used_by_discovery', 'true');
      this._addParameter(payload, 'patternMetadata', '{ "ciTypes" : [ "java.util.ArrayList", [ "cmdb_ci_docker_image" ] ], "relatedTables" : [ "java.util.ArrayList", [ "cmdb_key_value" ] ] }');
      this._addParameter(payload, 'prePatternExecutionData', '{ "fMapOfStrings" : { "image_name" : "' + docker_image_name + '", "image_sys_id" : "' + docker_image_sysid + '" }, "executePattern" : true, "empty" : false, "className" : "PrePatternExecutionDTO" }');
      this._addParameter(payload, 'computerSystem', '{ "ciSnapshotId" : 0, "computerIP" : "No Source", "aix" : false, "managementIP" : "No Source", "primaryManagementIP" : "No Source", "addressWidth" : 0, "hpux" : false, "solaris" : false }');
      return payload;
  },

   //Trivy command runs only Linux servers. So, created New Mid server capability "Scan Container Images" to specify which mids used for Scanning docker images	
  _findImageScanCapabilityMid: function() {
      var midSelector = new global.CloudMidSelectionApi();
      var midID = midSelector.selectMid(null, "Scan Container Images", null, "{}");
      if (gs.nil(midID))
          return;
      var midGr = new GlideRecord('ecc_agent');
      midGr.get(midID);
      var midName = midGr.name;
      this.logger.debug('select Mid is ' + midName + ' Mid Id= ' + midID);
      return midName;
  },


  _createDiscoveryStatus: function() {
      var newStatusgr = new GlideRecord(this._discovery_status_table);
      newStatusgr.initialize();
      newStatusgr.setValue("description", "Scan container images");
      newStatusgr.setValue("source", "scan_container_images");
      newStatusgr.setValue("discover", "ContainerImages");

      var statusId = newStatusgr.insert();
  	this.logger.debug(' Discovery Status Number ' + newStatusgr.number); 
      return (statusId) ? statusId : null;
  },

  _addParameter: function(payload, name, value) {
      var el = payload.createElement("parameter");
      el.setAttribute("name", name);

      if (value)
          el.setAttribute("value", value);

      return el;
  },

  _countInProgressImages: function() {
      //images scan_status is in in_progress and related discvery status is in Completed/Canceled then update scan_status to error.
      //Because, Pattern might have failed and not able to update the status.
      var imageScanGr = new GlideRecord(this._scan_status_table);
      imageScanGr.addQuery('scan_status', 'in_progress');
      imageScanGr.addQuery('discovery_status.state', 'IN', ['Completed', 'Canceled']);
      imageScanGr.setValue('scan_status', 'error');
      imageScanGr.updateMultiple();

      // Find In progrss scan_status images
      imageScanGr = new GlideAggregate(this._scan_status_table);
      imageScanGr.addQuery('scan_status', 'in_progress');
      imageScanGr.addAggregate('COUNT');
      imageScanGr.query();
      if (imageScanGr.next())
          return parseInt(imageScanGr.getAggregate('COUNT'));

      return 0;
  },

  _copyNextBatchImages: function() {

      var lastImageCopiedtime = this._getLastImageCopiedTime();
      this.logger.debug('Copying next batch images from ' + lastImageCopiedtime + ' timestamp');
      var lastimageTimestamp;
      var imageSysIds = [];
      var scanStatusSysIds = {};
      // Get Next batch images
      var imagesGr = new GlideRecord(this._oslv_image_table);
      imagesGr.addQuery('sys_created_on', '>=', lastImageCopiedtime);
      imagesGr.orderBy('sys_created_on');
      imagesGr.setLimit(2 * this._batchSize);
      imagesGr.query();
      while (imagesGr.next()) {
          lastimageTimestamp = imagesGr.sys_created_on + '';
          imageSysIds.push(imagesGr.getUniqueValue());
          this.logger.debug('Copying image ' + imagesGr.name + '   ' + imagesGr.getUniqueValue());
      }
  	if(imageSysIds.length ==0){
  		this.logger.debug(' No Images to copy');
  		return;
  	}
      // Get all Images created at lastimageTimestamp
      imagesGr = new GlideRecord(this._oslv_image_table);
      imagesGr.addQuery('sys_created_on', lastimageTimestamp);
      imagesGr.addQuery('sys_id', 'NOT IN', imageSysIds);
      imagesGr.query();
      while (imagesGr.next()) {
          imageSysIds.push(imagesGr.getUniqueValue());
  		this.logger.debug('Copying image ' + imagesGr.name + '   ' + imagesGr.getUniqueValue() + '  createdAt LastImageTimestamp');
      }
      // Check any next batch Images Already present in scan status table
      var imageScanStatusGr = new GlideRecord(this._scan_status_table);
      imageScanStatusGr.addQuery('image', 'IN', imageSysIds);
      imageScanStatusGr.query();
      while (imageScanStatusGr.next()) {
          scanStatusSysIds[imageScanStatusGr.image+''] = "true";
          this.logger.debug('image already present in scan_status table ' + imageScanStatusGr.image.name + '   ,Scan status sys_id =' + imageScanStatusGr.getUniqueValue());
      }

      //Insert Images into container Image Scan Status table 
      for (var i = 0; i < imageSysIds.length; i++) {
          if (typeof(scanStatusSysIds[imageSysIds[i]]) == 'undefined' || scanStatusSysIds[imageSysIds[i]] != 'true') {
              imageScanStatusGr.initialize();
              imageScanStatusGr.setValue('scan_status', 'none');
              imageScanStatusGr.setValue('image', imageSysIds[i]);
              imageScanStatusGr.insert();
              this.logger.debug('copied Image ' + imageScanStatusGr.image.name + '   ,Scan status sys_id =' + imageScanStatusGr.getUniqueValue());
          }
      }

      if (lastimageTimestamp) {
          var lastCreated = new GlideDateTime(lastimageTimestamp);
          // Add one second to lastimageTimestamp, so the next iteration will not pick same images again
          lastCreated.addSeconds(1);
          // Record the timestamp for the next iteration
          this._saveLastImagecopiedTime(lastCreated);
          this.logger.debug('Updated ' + this.LAST_CONTAINER_IMAGE_COPIED_TIME + ' in sa_hash table =' + lastCreated);
      } else {
          this.logger.debug('No change in the timestamp. Hence, Not Updated ' + this.LAST_CONTAINER_IMAGE_COPIED_TIME + ' in sa_hash table');
      }
  	this.logger.debug('Copy of next batch images completed');
  },


  // Record a timestamp to be used as input in the next iteration
  _saveLastImagecopiedTime: function(timestamp) {
      var grHash = new GlideRecord(this._sa_hash_table);
      grHash.addQuery('name', this.LAST_CONTAINER_IMAGE_COPIED_TIME);
      grHash.query();
      if (grHash.next()) {
          grHash.setValue('hash', timestamp);
          grHash.update();
      } else {
          grHash = new GlideRecord(this._sa_hash_table);
          grHash.setValue('name', this.LAST_CONTAINER_IMAGE_COPIED_TIME);
          grHash.setValue('hash', timestamp);
          grHash.insert();
      }

  },

  // Get the last saved timestamp recorded by the previous iteration
  _getLastImageCopiedTime: function() {
      var grHash = new GlideRecord(this._sa_hash_table);
      grHash.addQuery('name', this.LAST_CONTAINER_IMAGE_COPIED_TIME);
      grHash.query();
      if (grHash.next() && grHash.hash) {
          return new GlideDateTime(grHash.hash + '');
      } else {
          return new GlideDateTime('1970-01-01 12:00:00');
      }
  },

  type: 'ContainerImageScanJob'
};

Sys ID

7e90675e87a3d950b39eb846dabb351d

Offical Documentation

Official Docs: