Name

sn_agent.AgentUpgradeUtil

Description

No description available

Script

var AgentUpgradeUtil = Class.create();

AgentUpgradeUtil.debug = gs.getMessage('Debug');
AgentUpgradeUtil.info = gs.getMessage('Info');
AgentUpgradeUtil.warn = gs.getMessage('Warning');
AgentUpgradeUtil.error = gs.getMessage('Error');

AgentUpgradeUtil.prototype = {
  accfVersion: AgentVersionFieldStyle.getAccfVersion(),

  initialize: function() {},

  /* This method makes sure that the sn_agent_upgrade_history table does not grow too much by allowing a minimum number of of
  	 upgrades per agents to be saved at the table.
  */
  prepareTable: function(agentSysId) {
      var minNumberOfUpgradeHistoryTosSave = gs.getProperty('sn_agent.min_upgrade_history_per_agent', 2);

      var sysChoiceGr = new GlideRecord('sys_choice');
      sysChoiceGr.addQuery('name', 'sn_agent_upgrade_history');
      sysChoiceGr.addQuery('element', 'stage');
      sysChoiceGr.query();

      var numberOfStagesPerUpgrade = sysChoiceGr.getRowCount();

      var agentHistoryGr = new GlideRecord('sn_agent_upgrade_history');
      agentHistoryGr.addQuery('agent_sys_id', agentSysId);
      agentHistoryGr.orderBy('sys_updated_on');
      agentHistoryGr.query();

      /* Assuming the following:
      	minimum number of upgrade history per agent to save = 4
      	number of stages = 3
      	if we have more than 9 records, then start to delete records until we get to maximum of 9 records.
      		Why 9? because 9 is the max record count for 3 (4-1) upgrades.
      		So if we have more than 9 records - we definaly have at least 4 upgrades in the table
      		We can delete records until we will get to 3 upgrades in the table - then the currecnt upgrade will give us 4 upgrades
      */
      var maxNumberOfRecordsForMinNumberOfUpgradesMinusOne = numberOfStagesPerUpgrade * (minNumberOfUpgradeHistoryTosSave - 1);
      if (maxNumberOfRecordsForMinNumberOfUpgradesMinusOne >= agentHistoryGr.getRowCount()) {
          return;
      }

      var shouldContinue = true;
      var numberOfRows = agentHistoryGr.getRowCount();

      while (agentHistoryGr.next() && shouldContinue) {
          numberOfRows--;
          if (agentHistoryGr.getValue('stage') == 'InstanceVerification' && numberOfRows <= maxNumberOfRecordsForMinNumberOfUpgradesMinusOne) {
              shouldContinue = false;
          } else {
              agentHistoryGr.deleteRecord();
          }
      }
  },

  // This method generates a new unique sequence number for an agent and creates the initial stage
  // ('InstanceVerification') in 'In Progess' state
  startUpgradeSequence: function(agentSysId, agentVersion, msg) {
      var guid = gs.generateGUID();
      var upgradeGr = new GlideRecord('sn_agent_upgrade_history');
      upgradeGr.initialize();

      upgradeGr.setValue('unique_sequence', guid);

      upgradeGr.setValue('agent_sys_id', agentSysId);
      if (agentVersion)
          upgradeGr.setValue('from_version', agentVersion);
      upgradeGr.setValue('to_version', this.accfVersion);
      upgradeGr.setValue('stage', 'InstanceVerification');
      upgradeGr.setValue('state', 'InProgress');
      if (msg)
          upgradeGr.setValue('message', msg);

      upgradeGr.insert();

      return guid;
  },

  // This method creates a new stage for specific sequence
  createNewStateForSequence: function(agentSysId, sequenceId, agentVersion, stage, msg) {
      var upgradeGr = new GlideRecord('sn_agent_upgrade_history');
      upgradeGr.initialize();

      upgradeGr.setValue('unique_sequence', sequenceId);

      upgradeGr.setValue('agent_sys_id', agentSysId);
      if (agentVersion)
          upgradeGr.setValue('from_version', agentVersion);
      upgradeGr.setValue('to_version', this.accfVersion);
      upgradeGr.setValue('stage', stage);
      upgradeGr.setValue('state', 'InProgress');
      if (msg)
          upgradeGr.setValue('message', msg);

      return upgradeGr.insert();
  },

  // updates the specific upgrade sequence's stage with a new state
  updateSequenceStateByStage: function(agentSysId, sequenceId, state, stage, msg, concatenate) {
      var upgradeGr = new GlideRecord('sn_agent_upgrade_history');
      upgradeGr.addQuery('agent_sys_id', agentSysId);
      upgradeGr.addQuery('unique_sequence', sequenceId);
      upgradeGr.addQuery('stage', stage);
      upgradeGr.orderByDesc('sys_created_on');
      upgradeGr.setLimit(1);

      upgradeGr.query();
      if (upgradeGr.next()) {
          upgradeGr.setValue('state', state);
          if (msg) {
              if (concatenate)
                  msg = upgradeGr.getValue('message') + "\n" + msg;

              upgradeGr.setValue('message', msg);
          }

          upgradeGr.update();
          return upgradeGr.getUniqueValue();
      }

      return "";
  },

  // This method takes the latest stage that is 'In Progrss' for an agent and updates it with a new state.
  // It also, potentially updates the message
  updateLastInProgressByStage: function(agentSysId, state, stage, msg) {
      var upgradeGr = new GlideRecord('sn_agent_upgrade_history');
      upgradeGr.addQuery('agent_sys_id', agentSysId);
      upgradeGr.addQuery('stage', stage);
      upgradeGr.addQuery('state', 'InProgress');
      upgradeGr.orderByDesc('sys_created_on');
      upgradeGr.setLimit(1);

      upgradeGr.query();
      if (upgradeGr.next()) {
          upgradeGr.setValue('state', state);
          if (msg) {
              upgradeGr.setValue('message', msg);
          }

          upgradeGr.update();
          return upgradeGr.getUniqueValue();
      }

      return "";
  },

  // This method is called by the "Upgrade" Check Type and is processing the results of the Agent-Upgrade(s) Check Definitions
  // We expect results containing the Agent side verifications, this is why we update the 'AgentVerification' stage in this method
  processAgentUpgradeValidationResponse: function(checkResults) {
      var msg;
      if (!checkResults) {
          gs.error(gs.getMessage("Did not get any check results. Upgrade state unknown."));
          return;
      }

      for (var index = 0; index < checkResults.length; index++) {
          var check = checkResults[index]["check"];
          var upgradeSeqId = check["params_upgrade_seq"];
          if (!upgradeSeqId) {
              gs.error(gs.getMessage("Could not find sequence ID in check results. Check = {0}", JSON.stringify(check)));
              continue;
          }
          var agentId = checkResults[index]["agent_id"];

          var agentCiGr = new GlideRecord('sn_agent_cmdb_ci_agent');
          if (!agentCiGr.get('agent_id', agentId)) {
              msg = gs.getMessage('Could not locate agent with agent id {0}', agentId);
              gs.error(msg);
              return;
          }

          var agentSysId = agentCiGr.getUniqueValue();

          var agentExtendedGr = new GlideRecord('sn_agent_ci_extended_info');
          if (!agentExtendedGr.get('agent_id', agentId)) {
              this.exitWithError(agentSysId, upgradeSeqId, gs.getMessage("Could not locate agent id {0}. Upgrade state unknown.", agentId));
              return;
          }
          var agentVersion = agentExtendedGr.getValue('agent_version');

          if (!agentId || !check.output) {
              msg = gs.getMessage('Did not get enough details. Cannot continue with upgrade.');
              this.updateSequenceStateByStage(agentSysId, upgradeSeqId, 'Failed', 'AgentVerification', msg);
              return;
          }

          var outputLines = check.output.split("\n");

          if (outputLines[0] && outputLines[0].toLowerCase().indexOf('success') < 0) {
              var state = 'Failed';
              if (check.output && check.output.indexOf('is already installed') > 0)
                  state = 'Skipped';
              // We have failed !
              msg = gs.getMessage('Agent upgrade {0}. Check output: {1}', [state, check.output]);
              this.updateSequenceStateByStage(agentSysId, upgradeSeqId, state, 'AgentVerification', msg);
              return;
          }

          msg = gs.getMessage('Agent upgrade validation successful. Upgrade will now start and may take up to 30 minutes depand on the OS.');
          var checkResultsMessage = gs.getMessage("Check results");
          msg += "\n" + checkResultsMessage + ":\n" + check.output;
          this.updateSequenceStateByStage(agentSysId, upgradeSeqId, 'Success', 'AgentVerification', msg);
          msg = gs.getMessage('Agent upgrade is being performed.');
          this.createNewStateForSequence(agentSysId, upgradeSeqId, agentVersion, 'Upgrade', msg);

          agentExtendedGr.setValue('status', '4'); // Set to upgrading.
          agentExtendedGr.update();
      }
  },

  /* This method is called by the MonitoringConfig script include in the classifyClientsAndUpdate() method.
  	This method is called when an agent comes back to Up from Upgrading status.
  	After the agent is back up, we will invoke the "check-read-log" Check Definition to fetch the upgrade log file.
  */
  agentUpgradeToUp: function(agentId, isWindows) {
      var agentCiGr = new GlideRecord('sn_agent_cmdb_ci_agent');
      if (!agentCiGr.get('agent_id', agentId)) {
          gs.error(gs.getMessage("Could not locate agent record for agent id {0}", agentId));
      }
      var agentSysId = agentCiGr.getUniqueValue();

      var agentHistorySysId = this.updateLastInProgressByStage(agentSysId, 'Success', 'Upgrade', gs.getMessage("Fetching upgrade log"));

      var accFilePath = "$AGENT_LOG_ROOT/servicenow/agent-client-collector/upgrade.log";
      if (isWindows == 1) {
          accFilePath = "%TEMP%\\ACC_Logs\\ACC_Upgrade.log";
      }

      // Need to create a new glide record without the next() method to pass to the runCheckForCis() method
      agentCiGr = new GlideRecord('sn_agent_cmdb_ci_agent');
      agentCiGr.addQuery('agent_id', agentId);
      agentCiGr.query();
      var agentNow = new sn_agent.AgentNowHandler();
      // Fetch upgrade log file
      var check = {
          "checkDefId": "028fcd5067c80010b7b72dbd2685ef4f", // check-read-log
          "check_type_id": "754f5f84532d01100112ddeeff7b1248", // Upgrade log file
          "params": {
              "agentHistorySysId": agentHistorySysId,
              "file_path": accFilePath
          }
      };
      agentNow.runCheckForCis(agentCiGr, check, '1', 600000); // 10 minutes
  },

  /* This method is called from the Check Type "Upgrade log file".
     This method process the output of the upgrade log to analyze if the upgrade was successful or not.
  */
  processFetchedUpgradeLog: function(checkResults) {
      var msg;
      if (!checkResults) {
          gs.error(gs.getMessage("processFetchedUpgradeLog: Did not get any check results."));
          return;
      }

      for (var index = 0; index < checkResults.length; index++) {
          var check = checkResults[index]["check"];
          var agentId = checkResults[index]["agent_id"];
          var agentHistorySysId = check["params_agentHistorySysId"];
          if (!agentHistorySysId) {
              gs.error(gs.getMessage("processFetchedUpgradeLog: no history sys id."));
              continue;
          }

          var upgradeGr = new GlideRecord('sn_agent_upgrade_history');
          if (!upgradeGr.get('sys_id', agentHistorySysId)) {
              gs.error(gs.getMessage("processFetchedUpgradeLog: Did not locate history sys id."));
              continue;
          }

          outputLines = check.output.split("\n");
          var startCollectingLog = false;
          var log = "";
          var state = 'Failed';
          if (outputLines[outputLines.length - 1].toLowerCase().indexOf('upgrade success') > 0 || outputLines[outputLines.length - 2].toLowerCase().indexOf('upgrade success') > 0)
              state = 'Success';

          var upgradeFailed = false;
          /// Getting the relevant log lines specific for when the upgrade is done.
          for (var i = 0; i < outputLines.length; i++) {
              var line = outputLines[i];
              if (line.indexOf('Upgrade script started') >= 0)
                  startCollectingLog = true;

              if (!startCollectingLog)
                  continue;

              log += line + "\n";

              /* If the "Upgrade Success" line is unavailable, that means upgrade truly failed
              	or the logs were not retrieved in their entirety
              */
              if (state == 'Failed' && line.indexOf('Upgrade failed') > 0)
                  upgradeFailed = true;

              /* The line:
              		package agent-client-collector-1:2.9.0-1060.el7.x86_64 is already installed - is shown when the same version is installed.
              	The line:
              		package agent-client-collector-1:2.9.0-1060.el7.x86_64 (which is newer than agent-client-collector-1:2.8.0-1.el7.x86_64) is already installed - is shown when trying to downgrade
              */
              if (line.indexOf('is already installed') > 0)
                  state = 'Skipped';
          }

          if (state == 'Failed' && !upgradeFailed) {
              // Check if the agent is up and upgraded to the correct version before determining state
              var agentExtCiGr = new GlideRecord("sn_agent_ci_extended_info");
              if (agentExtCiGr.get('agent_id', agentId)
                  && agentExtCiGr.getValue("status") == "0"
                  && agentExtCiGr.getValue("agent_version") == this.accfVersion) {
                  state = 'Success';
                  log += "Agent is Up and upgraded to: " + this.accfVersion + ", but failed to fully fetch log.";
              } else
                  log += "Failed to fully fetch log. Unable to determine upgrade success. Please try again.";
          }

          upgradeGr.setValue('message', log);
          upgradeGr.setValue('state', state);
          upgradeGr.update();
      }
  },

  // This method is called from AgentKeepAlive script include.
  // If an agent was in Upgrading state for too long (timeout) then we will declare failed status on the upgrade
  markFailedIfTimeout: function(gr) {
      gr.query();
      var arrAgentsIds = [];
      while (gr.next()) {
          arrAgentsIds.push(gr.getValue('agent_id'));
      }

      if (arrAgentsIds.length < 1)
          return;

      var arrAgentSysIds = [];
      var agentCiGr = new GlideRecord('sn_agent_cmdb_ci_agent');
      agentCiGr.addQuery('agent_id', arrAgentsIds);
      agentCiGr.query();
      while (agentCiGr.next()) {
          arrAgentSysIds.push(agentCiGr.getUniqueValue());
      }

      var agentHistoryGr = new GlideRecord('sn_agent_upgrade_history');
      agentHistoryGr.addQuery('agent_sys_id', arrAgentSysIds);
      agentHistoryGr.addQuery('state', 'InProgress');
      agentHistoryGr.addQuery('stage', ['AgentVerification', 'Upgrade']);
      agentHistoryGr.setValue('state', 'Failed');
      agentHistoryGr.setValue('message', gs.getMessage("Timeout reached"));
      agentHistoryGr.updateMultiple();
  },

  exitWithError: function(agentSysId, upgradeSeqId, msg) {
      gs.error(msg);
      this.updateSequenceStateByStage(agentSysId, upgradeSeqId, 'Failed', 'AgentVerification', msg);
      return;
  },

  type: 'AgentUpgradeUtil'
};

Sys ID

20ead648532901100112ddeeff7b1251

Offical Documentation

Official Docs: