Name

global.EpSuggestions

Description

Guess entry points based on traffic based connections graph

Script

var EpSuggestions = Class.create();
EpSuggestions.prototype = {
  initialize: function() {
      this.procIds = [];
      this.procs = {};
      this.ip2node = {};
      this.candidates = [];
  },
  
  createNetstat : function() {
  	if (!GlidePluginManager.isActive('com.snc.service-mapping')) {
  	   gs.debug("Method call to createNetstat aborted since the Service Mapping plugin is not active");
  	   return;
  	}
  	var SOURCE = "netstat";
  	this.createNodes();
  	this.createEdges();
  	this.findCandidates();
  	this.writeCandidates(SOURCE);  
  	this.cleanupCandidatesWithEPs(SOURCE);
  },

  createNetflow: function() {
  	if (!GlidePluginManager.isActive('com.snc.service-mapping')) {
  	   gs.debug("Method call to createNetflow aborted since the Service Mapping plugin is not active");
  	   return;
  	}
      var SOURCE = "netflow";
      this.createGraphFlow();
      this.findCandidates();
      this.writeCandidates(SOURCE);
      this.cleanupCandidatesWithEPs(SOURCE);
  },

  createGraphFlow: function(){
      var flow = new GlideRecord("sa_flow_connection");
      flow.query();
      while (flow.next()) {
          var src = flow.ip + ":" + flow.port;
          var dst = flow.dest_ip + ":" + flow.dest_port;
          var srcNode = this.mkNode(src);
          var dstNode = this.mkNode(dst);            
          dstNode.inc.push({ip_port: dst, src: srcNode});
          srcNode.out.push({ip_port: dst, trg: dstNode});
      }
  },

  mkNode: function(id) {
      var node = this.procs[id];
      if (node == null) {
          node = {process: id, inc: [], out: [], lstn: [id]};
          this.procs[id] = node;
      }
      return node;
  },
  
  createNodes: function() {
  	
  	var MAX_ALLOWED_PROCS = parseInt(gs.getProperty("sa.traffic_based_discovery.ep_suggestions_max_allowed_procs", 200000)) || 200000;
      
      // Find all listeners which also have outgoing connections
      var tcpon = new GlideRecord("cmdb_tcp");
      tcpon.addQuery("type", "on");
      tcpon.addQuery("absent", false);

      // Do a self join to find all the processes that have 'connecting to' type as well
      var join = tcpon.addJoinQuery("cmdb_tcp", "process", "process");
      join.addCondition("type", "to");
      join.addCondition("absent", false);
      tcpon.query();

  	var numOfProcs = tcpon.getRowCount();
      gs.debug("found " + numOfProcs + " results.");
  	//to avoid excessive memory consumption, we are stopping the scheduled job that calls this script
  	//when the connecting processes number threshold is breached.
  	if (numOfProcs > MAX_ALLOWED_PROCS ) {
  		gs.debug("Processes number exceeded the allowed threshold. Deactivating the Generate entry point candidates scheduled job. To activate it again change the allowed procs threshold and activate the scheduled job.");
  		//deactivate the "Generate entry point candidates" scheduled job and stop the computation.
  		this.stopCandidatesScheduler();
  		return;
  	}	
  	
      // Populate the nodes data structures
      while (tcpon.next()) {
  		var node = {process: "" + tcpon.getValue("process"), computer: "" + tcpon.getValue("computer"), inc: [], out: [], lstn: []};
          var ip_port = tcpon.getValue("ip") + ":" + tcpon.getValue("port");
          if (this.procs[node.process] != null) 
              node = this.procs[node.process];
          else
              this.procs[node.process] = node;

          this.ip2node[ip_port] = node;
          node.lstn.push(ip_port);
      }

  	for (var proc in this.procs)
          this.procIds.push(proc);
  },
  
  stopCandidatesScheduler: function() {

  	//deactivate the scheduled job "Generate entry point candidates from netflow and netstat data"
  	var gr = new GlideRecord("sysauto_script");
  	gr.addQuery("sys_id", "b0b95067c3d0030039a3553a81d3aeb5");
  	gr.query();
  	if (gr.next()) {
  		gr.setValue('active', 'false');
  		gr.update();
  	}
  },

  createEdges: function() {

  	var MAX_CHUNK_SIZE = parseInt(gs.getProperty("sa.traffic_based_discovery.ep_suggestions_max_chunk_size", 1000)) || 1000;
  	var numOfProc = this.procIds.length;
  	// split the proc list into smaller chunks and call createEdgesByChunk
  	var ind = 0;
  	var procList = [];		
  	while (ind < numOfProc) {			
  		procList = this.procIds.slice(ind, ind + MAX_CHUNK_SIZE);
  		this.createEdgesByChunk(procList);
  		ind = ind + MAX_CHUNK_SIZE;
  	}
  },
  
  createEdgesByChunk: function(procList) {

  	// Find all the the outgoing connections from the processes with listening ports open
  	var tcpto = new GlideRecord("cmdb_tcp");
      tcpto.addQuery("type", "to");
      tcpto.addQuery("absent", false);
      tcpto.addQuery("process", "IN", procList);
      tcpto.query();
  	while (tcpto.next()) {          
          var src = this.procs["" + tcpto.process];            
          // todo: handle localhost
          var ip_port = tcpto.getValue("ip") + ":" +tcpto.getValue("port");
          
          var trg = this.ip2node[ip_port];
          if (src != null && trg != null) {
              trg.inc.push({ip_port: ip_port, src: src});
              src.out.push({ip_port: ip_port, trg: trg});
          }
      }
  },

  findCandidates: function() {
      // filter the process having http port and sort by # of incoming edges        
      var startNodes = [];
      // gs.print('procs are : ' + JSON.stringify(this.procs));
      for (var sys_id in this.procs) {
          var proc = this.procs[sys_id];
          gs.print('checking out proc: ' + proc.process);
          if (this.isValidEp(proc.lstn))
              startNodes.push(proc);
      }
      startNodes.sort(function(a, b) {
          return a.inc.length - b.inc.length;
      });

      gs.debug('got ' + startNodes.length + ' start nodes');
      for (var i = 0; i < startNodes.length; i++) {
          var node = startNodes[i];
          if (node.visited)
              continue;

          this.walk(node);
          gs.print('adding candidate: ' + JSON.stringify(node.lstn));
          this.candidates.push(node);
      }
  },

  walk: function(node) {
      // gs.print('walk: ' + JSON.stringify(node.lstn) + " visited: " + node.visited);
      if (node.visited)
          return;

      node.visited = true;
      for (var i = 0; i < node.out.length; i++)
          this.walk(node.out[i].trg);
  },

  writeCandidates: function(source) {
      var query = function(gr, vals) {
          for (var key in vals) {
              gr.addQuery(key, vals[key]);
          }
          return gr.query();
      };

      var update = function(gr, vals) {
          gr.setForceUpdate(true);
          for (var key in vals) {
              gr.setValue(key, vals[key]);
          }
          return gr.update();
      };

      var insert = function(gr, vals) {
          for (var key in vals) {
              gr.setValue(key, vals[key]);
          }
          return gr.insert();                
      };

      gs.debug('found ' + this.candidates.length + ' ep candidate processes');
      var gr, cand, ip_port, ip_port_spl, vals;
      for (var i = 0; i < this.candidates.length; i++) {
          cand = this.candidates[i];
          for (var j = 0; j < cand.lstn.length; j++) {
              ip_port = cand.lstn[j];
              if (this.isPortHttp(ip_port)) {
                  ip_port_spl = ip_port.split(":");                    
                  gr = new GlideRecord('sa_cand_entry_point');    
                  vals = { name : ip_port, ip_address: ip_port_spl[0], port: ip_port_spl[1], source: source };
                  query(gr, vals);
                  if (!gr.next()) { 
                      vals["ignore"] = false;
                      insert(gr, vals);
                  }
                  
              }
          }            
      }
  },

  isValidEp: function(list) {
      gs.print('listeners are ' + JSON.stringify(list));
      for (var i = 0; i < list.length; i++) {
          var ip_port = list[i];
          gs.print('ip_port ' + ip_port);
          if (this.isPortHttp(ip_port) && !this.isLoopback(ip_port))
              return true;
      }
      gs.print('invalid eps for this process');
      return false;
  },

  isLoopback: function(ip_port) {
      var startsWith = function(str, suffix) {
          return str.indexOf(suffix) == 0;
      };

      return startsWith(ip_port, "127.0.0.1:") || startsWith(ip_port, "0.0.0.0:");
  },

  isPortHttp: function(ip_port) {
      var endsWith = function(str, suffix) {
          return str.indexOf(suffix) + suffix.length == str.length;
      };
      return endsWith(ip_port, ":80") || endsWith(ip_port, ":443");
  },

  cleanupCandidatesWithEPs : function(source) {
      var gr = new GlideRecord('sa_cand_entry_point');    
      gr.addQuery('ignore', false);
      gr.addQuery('source', source);
      gr.addQuery('port', 80).addOrCondition('port', 443);
      var join = gr.addJoinQuery('cmdb_ci_endpoint', 'ip_address', 'ip_address');
      join.addCondition('port', 80).addOrCondition('port', 443);
      gr.query();
      while (gr.next()) {
          gs.debug('ignoring candidate ' + gr.name);
          gr.ignore = true;
          gr.update();
      }
  },

  type: 'EpSuggestions'
};

Sys ID

62d9fd89c3c0430039a3553a81d3ae1b

Offical Documentation

Official Docs: