Name
sn_agent.ProxyPolicyRedistribution
Description
Redistribution of monitoredCIs to different Proxy agents when Agent goes down/Agent comes up or Agent data_collection changed to off/on or agent added/removed/updated from cluster
Script
var ProxyPolicyRedistribution = Class.create();
ProxyPolicyRedistribution.prototype = {
initialize: function() {
this.policyStateUtil = new PolicyStateUtil();
},
/**
* Rearrange monitoredCIs to other agents as the passed "downAgentIds" went down
* @param policy
* @param downAgentIds array of agent_ids that went down
*/
rearrangeMonitoredCIsOnAgentDown: function(policy, downAgentIds) {
gs.debug("ProxyPolicyRedistribution: rearrangeMonitoredCIstoProxyAgents called with policy=" + policy + " and down agent=" + downAgentIds);
var monitoredCisPerAgent = {};
var agg = new GlideAggregate('sn_agent_policy_monitored_cis');
//If any of the agent are down and data_collection is off, don't redistribute to those agents, so filter them
agg.addEncodedQuery("agent_id.statusIN0,1^agent_id.data_collection=0");
// Count how many times does our agent is shown per policy = how many monitored CIs per agent
agg.addAggregate('COUNT', 'agent_id');
agg.addQuery('agent_id', 'NOT IN', downAgentIds);
agg.addQuery('policy', policy);
// Cannot sort by GlideAggregate, will sort manually.
agg.query();
//TODO: there are no agents and everything went down. so kept the existing entries as it is
if (agg.getRowCount() == 0) {
return;
}
// Build a map of agent id to number of monitored CIs.
while (agg.next()) {
monitoredCisPerAgent[agg.getValue('agent_id')] = agg.getAggregate('COUNT', 'agent_id');
}
var sortedMonitoredCisPerAgent = [];
for (var agentId in monitoredCisPerAgent) {
sortedMonitoredCisPerAgent.push([agentId, monitoredCisPerAgent[agentId]]);
}
// End results is a sorted map.
sortedMonitoredCisPerAgent.sort(function(a, b) {
return a[1] - b[1];
});
//used sorted agents so that monitoredCIs are allocated to the least load agent first.
var sortedProxyAgents = [];
for (var i = 0; i < sortedMonitoredCisPerAgent.length; i++) {
sortedProxyAgents.push(sortedMonitoredCisPerAgent[i][0]); // take only the agent ID from the map
}
//query monitored cis that are assigned to down agent and redistribute to other agents
var monitoredCIsToRearrange = new GlideRecord('sn_agent_policy_monitored_cis');
// monitoredCIsToRearrange.setWorkflow(false); // do not trigger BRs
monitoredCIsToRearrange.addQuery('agent_id', "IN", downAgentIds);
monitoredCIsToRearrange.addQuery('policy', policy);
monitoredCIsToRearrange.query();
var sortedProxyAgentsIndex = 0;
var sortedProxyAgentsLength = sortedProxyAgents.length;
while (monitoredCIsToRearrange.next()) {
gs.debug("ProxyPolicyRedistribution: Assigning monitoredCI " + monitoredCIsToRearrange.monitored_ci + " to agent_id to " + sortedProxyAgents[sortedProxyAgentsIndex % sortedProxyAgentsLength].toString().trim() + " from agent_id is" + monitoredCIsToRearrange.agent_id);
monitoredCIsToRearrange.setValue('agent_id', '' + sortedProxyAgents[sortedProxyAgentsIndex++ % sortedProxyAgentsLength]);
monitoredCIsToRearrange.update();
}
},
/**
* This method is called when agents goes down to redistribute the monitoredCIs associated to them to other agents
* @param downAgentIds comma seperated agent_ids of agents that went down.
*/
onAgentDown: function(downAgentIds) {
var policiesToBeUpdated = {};
var policySysId;
var agent_id;
gs.info("ProxyPolicyRedistribution: Calling Redistribute MonitoredCIs for Proxy policies as these agents went down:" + downAgentIds);
//Load balancing usecase where run_checks_on_all_proxyagents=false
//check whether the agents that went down are used by any policy as proxy agent that is either single proxy agent, proxy agent by filter, proxy agent by cluster.
var mappingGr = new GlideRecord("sn_agent_policy_monitored_cis");
mappingGr.addQuery("agent_id", "IN", downAgentIds);
//Call redistribution only when load balancing is on and not running the check on all agents.
mappingGr.addQuery("policy.run_checks_on_all_proxyagents", "false");
mappingGr.addQuery("policy.single_proxy_agent", "true")
.addOrCondition("policy.proxy_advanced", "true")
.addOrCondition("policy.proxy_cluster", "true");
mappingGr.query();
while (mappingGr.next()) {
policySysId = mappingGr.getValue('policy');
agent_id = mappingGr.getValue('agent_id');
//If policy and agent_id combination is not already added, add it to the policiesToBeUpdated object, as we have multiple entries for same policy with same agent_id with different monitoredCIs,
// as agent may be monitoring multiple monitoredCIs of the same policy
//If policy is not already added, add it to the array
if (!policiesToBeUpdated[policySysId])
policiesToBeUpdated[policySysId] = [];
//If agent_id for that policy doesn't exist add it, if it exists dont add duplicate entries for same agent_id
if (policiesToBeUpdated[policySysId].indexOf(agent_id) == -1)
policiesToBeUpdated[policySysId].push(agent_id);
}
for (var policySysId in policiesToBeUpdated) {
var arrAgents = policiesToBeUpdated[policySysId];
if (arrAgents && arrAgents.length > 0)
this.rearrangeMonitoredCIsOnAgentDown(policySysId, arrAgents);
}
//for all policies set publish_status = 5 "Ready to publish" which tells the system to publish this policy to the MID server.
var policyClientsGeneratorNG = new PolicyClientsGeneratorNG(this.policyStateUtil);
policyClientsGeneratorNG.resetPoliciesToClientsForMidSync(Object.keys(policiesToBeUpdated));
//STRY54809380: For Policies with Advanced script need to do full recalculation
//Load balancing usecase where run_checks_on_all_proxyagents=false and for proxy policies with advanced script option
var policiesNeedFullRecalc = [];
mappingGr = new GlideRecord("sn_agent_policy_monitored_cis");
mappingGr.addQuery("agent_id", "IN", downAgentIds);
//Call redistribution only when load balancing is on and not running the check on all agents.
mappingGr.addQuery("policy.run_checks_on_all_proxyagents", "false");
mappingGr.addQuery("policy.proxy_script_advanced", "true");
mappingGr.query();
while (mappingGr.next()) {
policySysId = mappingGr.getValue('policy');
if (policiesNeedFullRecalc.indexOf(policySysId) == -1)
policiesNeedFullRecalc.push(policySysId);
}
//If there are policies with proxy advanced script that is using agent that went down, need to do full recalculation
if (policiesNeedFullRecalc.length > 0) {
//get all the proxy policies where load balancing is enabled and this downAgent is used for monitoring
var proxyPolicyGr = new GlideRecord('sn_agent_policy');
proxyPolicyGr.addActiveQuery();
proxyPolicyGr.addQuery('is_draft', '0');
proxyPolicyGr.addQuery("proxy_script_advanced", "true");
proxyPolicyGr.addQuery("sys_id", 'IN', policiesNeedFullRecalc);
proxyPolicyGr.query();
while (proxyPolicyGr.next()) {
var publishStatus = this.policyStateUtil.getPolicyStatePublishStatus(proxyPolicyGr);
var lastFullCalcTime = this.policyStateUtil.getPolicyStateLastFullCalcTime(proxyPolicyGr);
if (publishStatus == '3' || publishStatus == '4' || !lastFullCalcTime)
continue;
// set publish status=3 is a Queued, put the policy to Queued state so that it will do full recalculation.
policyClientsGeneratorNG.setPolicyPublishQueued(proxyPolicyGr);
gs.debug("ProxyPolicyRedistribution:proxy_script_advanced policy: changing policy state to Queued for policy=" + proxyPolicyGr.name + " as one of agent in policy went down");
}
}
},
/**
* This method is called when data_collection for agent is off
* @param dataCollectionOffAgentIds comma seperated agent_ids of the agents where data_collection is off, MonitoredCIs monitored by the dataCollectionOffAgentIds has to redistributed to different agents
*/
onAgentSilent: function(dataCollectionOffAgentIds) {
gs.info("ProxyPolicyRedistribution: Calling Redistribute MonitoredCIs for Proxy policies as these agents data_collection is off:" + dataCollectionOffAgentIds);
this.onAgentDown(dataCollectionOffAgentIds);
},
/**
* This method is called when data_collection for agent goes on
* @param dataCollectionOnAgentIds comma seperated agent_ids of the agents where data_collection becomes on, MonitoredCIs monitored by other agents has to be redistributed to these up agents
*/
onAgentDataCollectionOn: function(dataCollectionOnAgentIds) {
gs.info("ProxyPolicyRedistribution: Calling Redistribute MonitoredCIs for Proxy policies as these agents data_collection is on:" + dataCollectionOnAgentIds);
this.onAgentUp(dataCollectionOnAgentIds);
},
/**
* This method is called when agent comesup
* @param upAgentIds comma seperated agent_ids of the agents that has come up, MonitoredCIs monitored by different agents can be given to this agent
*/
onAgentUp: function(upAgentIds) {
var arrUPAgentIds = upAgentIds.split(",");
var policyClientsGeneratorNG = new PolicyClientsGeneratorNG(this.policyStateUtil);
var agent_id;
// get only those agents that has data_collection on, even if the up agents has data_Collection off dont consider them.
var agentsGR = new GlideRecord('sn_agent_cmdb_ci_agent');
//take only proxy that is up and not silent
agentsGR.addEncodedQuery("agent_extended_info.statusIN0,1^agent_extended_info.data_collection=0");
agentsGR.addQuery('agent_id', "IN", arrUPAgentIds);
agentsGR.query();
//filter from upAgents whose data_collection is off and take only agents that data_Collection on
arrUPAgentIds = [];
while (agentsGR.next()) {
arrUPAgentIds.push(agentsGR.getValue('agent_id'));
}
gs.info("ProxyPolicyRedistribution: Calling Redistribute MonitoredCIs for Proxy policies as these agents comes UP:" + upAgentIds);
var policyRecalcRequired = false; //to know whether the current policy is using one of the upAgents.
//check for all policies where this array of agents is used
var proxyPolicyGr = new GlideRecord('sn_agent_policy');
proxyPolicyGr.addActiveQuery();
proxyPolicyGr.addQuery('is_draft', '0');
proxyPolicyGr.addQuery("single_proxy_agent", "true")
.addOrCondition("proxy_advanced", "true")
.addOrCondition("proxy_script_advanced", "true")
.addOrCondition("proxy_cluster", "true");
proxyPolicyGr.query();
while (proxyPolicyGr.next()) {
var publishStatus = this.policyStateUtil.getPolicyStatePublishStatus(proxyPolicyGr);
var lastFullCalcTime = this.policyStateUtil.getPolicyStateLastFullCalcTime(proxyPolicyGr);
if (publishStatus == '3' || publishStatus == '4' || !lastFullCalcTime)
continue;
policyRecalcRequired = false;
//single proxy agent policy
if (proxyPolicyGr.single_proxy_agent) {
agent_id = proxyPolicyGr.proxy_agent.agent_id.toString();
if (arrUPAgentIds.indexOf(agent_id) != -1) {
policyRecalcRequired = true;
}
} else if (proxyPolicyGr.proxy_cluster) { //policy with proxy cluster
//check whether the agents are present in the proxy_Cluster of this policy
var agentclusterGR = new GlideRecord('sn_agent_in_agentcluster');
agentclusterGR.addQuery("cluster", proxyPolicyGr.agent_cluster_name);
agentclusterGR.addQuery("agent.agent_id", "IN", arrUPAgentIds);
agentclusterGR.setLimit(1);
agentclusterGR.query();
if (agentclusterGR.hasNext()) {
if (gs.isDebugging()) {
agentclusterGR.next();
gs.debug("ProxyPolicyRedistribution: Calling Redistribute for policy=" + proxyPolicyGr.name + " as agents in this cluster has come up agentclusterGR=" + agentclusterGR.cluster.name);
}
policyRecalcRequired = true;
}
} else if (proxyPolicyGr.proxy_advanced) { //proxy agent with filter option
var agentsGR = new GlideRecord('sn_agent_cmdb_ci_agent');
if (proxyPolicyGr.proxy_filter) {
agentsGR.addEncodedQuery(proxyPolicyGr.proxy_filter);
}
agentsGR.addQuery('agent_id', "IN", arrUPAgentIds);
agentsGR.setLimit(1);
agentsGR.query();
if (agentsGR.hasNext()) {
policyRecalcRequired = true;
}
} else if (proxyPolicyGr.proxy_script_advanced) { //proxy selection with advanced script
//STRY54809380: For Policies with Advanced script need to do full recalculation if the agent that cameup is used by any monitoredCI
//check whether these agents are used by any of monitoredCIs in the proxy advanced script
policyRecalcRequired = false; //false by default
var monitoredCisGr = this.getMonitoredCIsGrForPolicy(proxyPolicyGr);
if (monitoredCisGr != null) {
monitoredCisGr.setCategory('acc-policy-calc');
monitoredCisGr.query();
var tableName = monitoredCisGr.getTableName();
var isEntryPointView = (tableName == policyClientsGeneratorNG.ENTRYPOINT_DB_VIEW);
var fieldToGetSysId = 'sys_id';
if (isEntryPointView)
fieldToGetSysId = 'endpoint_sys_id';
while (monitoredCisGr.next()) {
var monitoredCiSysId = monitoredCisGr.getValue(fieldToGetSysId);
// Perform the script associated with the proxy policy per monitored CI (this is how this feature was designed)
var evaluator = new GlideScopedEvaluator();
evaluator.putVariable('answer', null);
evaluator.putVariable('currentCiResult', monitoredCisGr); //pass current host record as param
var proxy_agents = evaluator.evaluateScript(proxyPolicyGr, 'proxy_script');
if (!proxy_agents)
proxy_agents = evaluator.getVariable('answer');
// Validate that proxy_agents var is of type string
if (typeof proxy_agents != 'string') {
gs.warn(gs.getMessage("ProxyPolicyRedistribution: proxy_agents did not evaluate to type string, cannot process proxy for CI {0} on policy {1}. proxy_agents type: {2}",
[monitoredCiSysId, proxyPolicyGr.getValue('name'), typeof proxy_agents]));
continue; //to check whether agent is used for other monitoredCI
}
var proxyAgents = proxy_agents.split(/\s*,\s*/); // Split, ignore spaces.
if (!proxyAgents || proxyAgents.length < 1) {
gs.warn(gs.getMessage("ProxyPolicyRedistribution: did not get any proxy agents for CI {0} on policy {1}. Cannot contine",
[monitoredCiSysId, proxyPolicyGr.getValue('name')]));
continue; //to check whether agent is used for other monitoredCI
}
//arrUPAgentIds
for (var index = 0; index < proxyAgents.length; index++) {
agent_id = proxyAgents[index];
if (arrUPAgentIds.indexOf(agent_id) != -1) {
policyRecalcRequired = true;
// break this forloop as one of the monitoredCI is using one of the upagents for monitoring.
break;
}
}
if (policyRecalcRequired) {
break; // break this while loop and put the policy for full recalculation as the upagents can be used as part of the policy monitoring.
//and need to check for other monitoredCIs as its alreayd getting used and full policy recalculation has to happen
}
}
}
}
if (policyRecalcRequired) {
//put the policy to Queued state so that it will do full recalculation.
policyClientsGeneratorNG.setPolicyPublishQueued(proxyPolicyGr);
}
}
},
/**
* Method called when proxy agents are added/deleted from cluster to do full recalculation for policies that use this cluster
* @param proxyAgentClusterSysId
*/
onProxyClusterChange: function(proxyAgentClusterSysId) {
var policyClientsGeneratorNG = new PolicyClientsGeneratorNG(this.policyStateUtil);
//get all the proxy policies where load balancing is enabled to check whether this agent is used by these policies and redistribute CIs
var proxyPolicyGr = new GlideRecord('sn_agent_policy');
proxyPolicyGr.addActiveQuery();
proxyPolicyGr.addQuery('is_draft', '0');
proxyPolicyGr.addQuery("proxy_cluster", "true");
proxyPolicyGr.addQuery("agent_cluster_name", proxyAgentClusterSysId);
proxyPolicyGr.query();
while (proxyPolicyGr.next()) {
var publishStatus = this.policyStateUtil.getPolicyStatePublishStatus(proxyPolicyGr);
var lastFullCalcTime = this.policyStateUtil.getPolicyStateLastFullCalcTime(proxyPolicyGr);
if (publishStatus == '3' || publishStatus == '4' || !lastFullCalcTime)
continue;
// set publish status=3 is a Queued, put the policy to Queued state so that it will do full recalculation.
policyClientsGeneratorNG.setPolicyPublishQueued(proxyPolicyGr);
gs.debug("ProxyPolicyRedistribution:onProxyClusterChange: changing policy state to Queued for policy=" + proxyPolicyGr.name + " as cluster " + proxyPolicyGr.agent_cluster_name.getDisplayValue() + " has changed");
}
},
/**
* For proxy script policies need to know the MonitoredCIs, to check whether the agent that cameup will be used by any monitoredCI or not.
* @param policyGr
* @returns {GlideRecord|null}
*/
getMonitoredCIsGrForPolicy: function(policyGr) {
var policyClientsGeneratorNG = new PolicyClientsGeneratorNG(this.policyStateUtil);
var monitoredCisGr = "";
var monitoredCiType = "";
var cisFromService = "";
var evaluator = new GlideScopedEvaluator();
if (policyGr.monitored_ci_type_script) {
monitoredCisGr = evaluator.evaluateScript(policyGr, 'monitored_ci_script');
if (!monitoredCisGr) {
var msg = gs.getMessage("ProxyPolicyRedistribution: Policy {0}, the monitored CI type based on the script has failed. Check your script and try again.", policyGr.name);
gs.error("getMonitoredCIsGRForPolicy: While looking for Policies with up agents, : " + msg);
return null;
}
if (monitoredCisGr) {
try {
// We are setting a limit to make sure the script did not go more than we want it too.
monitoredCisGr.setLimit(policyClientsGeneratorNG.MAX_NUMBER_OF_MONITORED_CIS_TO_PROCESS);
monitoredCisGr.setCategory('acc-policy-calc');
monitoredCisGr.query();
} catch (e) {
gs.error(gs.getMessage("ProxyPolicyRedistribution: For policy {0}, the script to get the monitored CIs had failed. Try fixing it. Error={1}", [policyGr.name, e]));
return null;
}
monitoredCiType = monitoredCisGr.getTableName();
cisFromService = undefined;
// If service query, add the syd ids of the CIs in the service.
// Assumption here is that the service CIs count is a relativley low number
// so it is good practice to add it to the query at this point
if (policyGr.is_service_filter) {
cisFromService = policyClientsGeneratorNG.getCisForServiceByServiceQuery(policyGr, monitoredCiType);
if (cisFromService)
monitoredCisGr.addQuery("sys_id", cisFromService);
}
}
return monitoredCisGr;
} else if (policyGr.monitored_ci_type_group) {
var groupName = policyGr.monitored_ci_group;
var groupRecord = new GlideRecord("cmdb_group");
groupRecord.addQuery("group_name", groupName);
groupRecord.query();
if (!groupRecord.next()) {
gs.error(gs.getMessage("ProxyPolicyRedistribution: getMonitoredCIsGrForPolicy: in policyCalculationForMonitoredCisByGroup(), could not find any group named {0}", groupName));
return null;
}
var groupId = groupRecord.getValue("sys_id");
var results = JSON.parse(sn_cmdbgroup.CMDBGroupAPI.getAllCI(groupId));
if (!results || !results.idList || results.idList.length < 1) {
return null;
}
var ciRecord = new GlideRecord("cmdb_ci");
if (!ciRecord.get('sys_id', results.idList[0])) {
gs.error(gs.getMessage('ProxyPolicyRedistribution: getMonitoredCIsGrForPolicy:: For policy {1}, something went wrong when trying to get data for {1}', [policyGr.name, ciRecord]));
return null;
}
var className = ciRecord.getValue("sys_class_name");
//find the most generic type that suits this class type
var commonTable = policyClientsGeneratorNG.getParentTable(className);
var commonTableNoUnderscore = commonTable.replace(/_/g, '');
cisFromService = undefined;
// If service query, add the syd ids of the CIs in the service.
// Assumption here is that the service CIs count is a relativley low number
// so it is good practice to add it to the query at this point
if (policyGr.is_service_filter) {
cisFromService = policyClientsGeneratorNG.getCisForServiceByServiceQuery(policyGr, commonTable);
if (!cisFromService || cisFromService.length < 1) {
gs.error(gs.getMessage("ProxyPolicyRedistribution: getMonitoredCIsGrForPolicy::: Policy {0} filters based on application service which returned 0 CIs for the CI type {1}. Policy has no agents", [policyGr.name, commonTable]));
return null;
}
}
// This is a proxy policy (the agents are being selected by a different mechanism, and no need for a db view table)
monitoredCisGr = new GlideRecord(commonTable);
monitoredCisGr.addQuery("sys_id", "IN", results.idList); // NEED TO REPLACE WITH CHOOSE WINDOW AND CHUNKS
if (cisFromService)
monitoredCisGr.addQuery("sys_id", cisFromService);
return monitoredCisGr;
} else {
monitoredCiType = policyGr.getValue('table');
var filter = policyGr.getValue('filter');
if (!filter)
filter = "";
cisFromServices = undefined;
// If service query, add the syd ids of the CIs in the service.
// Assumption here is that the service CIs count is a relativley low number
// so it is good practice to add it to the query at this point
if (policyGr.is_service_filter) {
cisFromServices = policyClientsGeneratorNG.getCisForServiceByServiceQuery(policyGr, monitoredCiType);
if (!cisFromServices || cisFromServices.length < 1) {
gs.info(gs.getMessage("ProxyPolicyRedistribution: Policy {0} filters based on application service which returned 0 CIs for the CI type {1}. Policy has no agents", [policyGr.name, monitoredCiType]));
return null;
}
}
monitoredCisGr = new GlideRecord(monitoredCiType);
monitoredCisGr.addEncodedQuery(filter);
if (cisFromServices)
monitoredCisGr.addQuery("sys_id", cisFromServices);
return monitoredCisGr;
}
},
type: 'ProxyPolicyRedistribution'
};Sys ID
6af2f6161be71110725721b8b04bcb50