AMBARI-13210: RU - Install version stuck (jluniya)

ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py

@@ -18,6 +18,7 @@ limitations under the License.
 
 """
 
+import os
 import sys
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
@@ -27,6 +28,7 @@ from resource_management.libraries.functions.get_hdp_version import get_hdp_vers
 from resource_management.libraries.script.script import Script
 from resource_management.core.shell import call
 from resource_management.libraries.functions.version import format_hdp_stack_version
+from resource_management.libraries.functions.version_select_util import get_versions_from_stack_root
 
 HDP_SELECT_PREFIX = ('ambari-python-wrap', 'hdp-select')
 # hdp-select set oozie-server 2.2.0.0-1234
@@ -240,12 +242,19 @@ def _get_upgrade_stack():
   return None
 
 
-def get_hdp_versions():
+def get_hdp_versions(stack_root):
+  """
+  Gets list of stack versions installed on the host.
+  Be default a call to hdp-select versions is made to get the list of installed stack versions.
+  As a fallback list of installed versions is collected from stack version directories in stack install root.
+  :param stack_root: Stack install root
+  :return: Returns list of installed stack versions.
+  """
   code, out = call(HDP_SELECT_PREFIX + ('versions',))
+  versions = []
   if 0 == code:
-    versions = []
     for line in out.splitlines():
       versions.append(line.rstrip('\n'))
-    return versions
-  else:
-    return []
+  if not versions:
+    versions = get_versions_from_stack_root(stack_root)
+  return versions

ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py

@@ -19,6 +19,7 @@ limitations under the License.
 Ambari Agent
 
 """
+import os
 import re
 import tempfile
 
@@ -74,4 +75,21 @@ def get_component_version(stack_name, component_name):
   else:
     Logger.error("Could not find a stack for stack name: %s" % str(stack_name))
 
-  return version
+  return version
+
+
+def get_versions_from_stack_root(stack_root):
+  """
+  Given a stack install root (/usr/hdp), returns a list of stack versions currently installed.
+  The list of installed stack versions is determined purely based on the stack version directories
+  found in the stack install root.
+  Because each stack name may have different logic, the input is a generic dictionary.
+  :param stack_root: Stack install root directory
+  :return: Returns list of installed stack versions
+  """
+  if stack_root is None or not os.path.exists(stack_root):
+    return []
+
+  installed_stack_versions = [f for f in os.listdir(stack_root) if os.path.isdir(os.path.join(stack_root, f))
+                              and re.match("([\d\.]+(-\d+)?)", f)]
+  return installed_stack_versions

ambari-server/src/main/java/org/apache/ambari/server/events/listeners/upgrade/DistributeRepositoriesActionListener.java

@@ -99,20 +99,23 @@ public class DistributeRepositoriesActionListener {
     String repositoryVersion = null;
 
     if (event.getCommandReport() == null) {
-      LOG.error("Command report is null, will set all INSTALLING versions for host {} to INSTALL_FAILED.", event.getHostname());
+      LOG.error(
+          "Command report is null, will set all INSTALLING versions for host {} to INSTALL_FAILED.",
+          event.getHostname());
+    } else if (!event.getCommandReport().getStatus().equals(HostRoleStatus.COMPLETED.toString())) {
+      LOG.warn(
+          "Distribute repositories did not complete, will set all INSTALLING versions for host {} to INSTALL_FAILED.",
+          event.getHostname());
     } else {
       // Parse structured output
       try {
+        newHostState = RepositoryVersionState.INSTALLED;
         DistributeRepositoriesStructuredOutput structuredOutput = StageUtils.getGson().fromJson(
                 event.getCommandReport().getStructuredOut(),
                 DistributeRepositoriesStructuredOutput.class);
 
         repositoryVersion = structuredOutput.getInstalledRepositoryVersion();
 
-        if (event.getCommandReport().getStatus().equals(HostRoleStatus.COMPLETED.toString())) {
-          newHostState = RepositoryVersionState.INSTALLED;
-        }
-
         // Handle the case in which the version to install did not contain the build number,
         // but the structured output does contain the build number.
         if (null != structuredOutput.getActualVersion() && !structuredOutput.getActualVersion().isEmpty() &&

ambari-server/src/main/resources/custom_actions/scripts/install_packages.py

@@ -88,9 +88,12 @@ class InstallPackages(Script):
           self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[stack_name]
     if self.stack_root_folder is None:
       raise Fail("Cannot determine the stack's root directory by parsing the stack_id property, {0}".format(str(stack_id)))
+    if self.repository_version is None:
+      raise Fail("Cannot determine the repository version to install")
 
     self.repository_version = self.repository_version.strip()
 
+
     # Install/update repositories
     installed_repositories = []
     self.current_repositories = []
@@ -130,24 +133,10 @@ class InstallPackages(Script):
     if num_errors > 0:
       raise Fail("Failed to distribute repositories/install packages")
 
-    # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
-    # to correct value if it is not
-    self.actual_version = None
-    if self.repository_version:
-      m = re.search("[\d\.]+-\d+", self.repository_version)
-      if m:
-        # Contains a build number
-        self.repo_version_with_build_number = self.repository_version
-        self.structured_output['actual_version'] = self.repo_version_with_build_number  # This is the best value known so far.
-        self.put_structured_out(self.structured_output)
-      else:
-        self.repo_version_with_build_number = None
-
     # Initial list of versions, used to compute the new version installed
-    self.old_versions = get_hdp_versions()
+    self.old_versions = get_hdp_versions(self.stack_root_folder)
 
     try:
-      # It's possible for the process to receive a SIGTERM while installing the packages
       ret_code = self.install_packages(package_list)
       if ret_code == 0:
         self.structured_output['package_installation_result'] = 'SUCCESS'
@@ -169,18 +158,31 @@ class InstallPackages(Script):
 
   def compute_actual_version(self):
     """
-    After packages are installed, determine what the new actual version is, in order to save it.
+    After packages are installed, determine what the new actual version is.
     """
+
+    # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
+    # to correct value if it is not
+    self.actual_version = None
+    self.repo_version_with_build_number = None
+    if self.repository_version:
+      m = re.search("[\d\.]+-\d+", self.repository_version)
+      if m:
+        # Contains a build number
+        self.repo_version_with_build_number = self.repository_version
+        self.structured_output['actual_version'] = self.repo_version_with_build_number  # This is the best value known so far.
+        self.put_structured_out(self.structured_output)
+
     Logger.info("Attempting to determine actual version with build number.")
     Logger.info("Old versions: {0}".format(self.old_versions))
 
-    new_versions = get_hdp_versions()
+    new_versions = get_hdp_versions(self.stack_root_folder)
     Logger.info("New versions: {0}".format(new_versions))
 
     deltas = set(new_versions) - set(self.old_versions)
     Logger.info("Deltas: {0}".format(deltas))
 
-    # Get HDP version without build number
+    # Get version without build number
     normalized_repo_version = self.repository_version.split('-')[0]
 
     if 1 == len(deltas):
@@ -188,21 +190,92 @@ class InstallPackages(Script):
       self.structured_output['actual_version'] = self.actual_version
       self.put_structured_out(self.structured_output)
       write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
+      Logger.info(
+        "Found actual version {0} by checking the delta between versions before and after installing packages".format(
+          self.actual_version))
     else:
-      Logger.info("Cannot determine a new actual version installed by using the delta method.")
       # If the first install attempt does a partial install and is unable to report this to the server,
-      # then a subsequent attempt will report an empty delta. For this reason, it is important to search the
-      # repo version history file to determine if we previously did write an actual_version.
-      self.actual_version = read_actual_version_from_history_file(normalized_repo_version)
+      # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
+      Logger.info("Cannot determine actual version installed by checking the delta between versions "
+                  "before and after installing package")
+      Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
+      self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
       if self.actual_version is not None:
         self.actual_version = self.actual_version.strip()
         self.structured_output['actual_version'] = self.actual_version
         self.put_structured_out(self.structured_output)
-        Logger.info("Found actual version {0} by parsing file {1}".format(self.actual_version, REPO_VERSION_HISTORY_FILE))
-      elif self.repo_version_with_build_number is None:
+        Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
+      else:
         msg = "Could not determine actual version installed. Try reinstalling packages again."
         raise Fail(msg)
 
+  def check_partial_install(self):
+    """
+    If an installation did not complete successfully, check if installation was partially complete and
+    log the partially completed version to REPO_VERSION_HISTORY_FILE.
+    :return:
+    """
+    Logger.info("Installation of packages failed. Checking if installation was partially complete")
+    Logger.info("Old versions: {0}".format(self.old_versions))
+
+    new_versions = get_hdp_versions(self.stack_root_folder)
+    Logger.info("New versions: {0}".format(new_versions))
+
+    deltas = set(new_versions) - set(self.old_versions)
+    Logger.info("Deltas: {0}".format(deltas))
+
+    # Get version without build number
+    normalized_repo_version = self.repository_version.split('-')[0]
+
+    if 1 == len(deltas):
+      # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
+      partial_install_version = next(iter(deltas)).strip()
+      write_actual_version_to_history_file(normalized_repo_version, partial_install_version)
+      Logger.info("Version {0} was partially installed. ".format(partial_install_version))
+
+  def find_best_fit_version(self, versions, repo_version):
+    """
+    Given a list of installed versions and a repo version, search for a version that best fits the repo version
+    If the repo version is found in the list of installed versions, return the repo version itself.
+    If the repo version is not found in the list of installed versions
+    normalize the repo version and use the REPO_VERSION_HISTORY_FILE file to search the list.
+
+    :param versions: List of versions installed
+    :param repo_version: Repo version to search
+    :return: Matching version, None if no match was found.
+    """
+    if versions is None or repo_version is None:
+      return None
+
+    build_num_match = re.search("[\d\.]+-\d+", repo_version)
+    if build_num_match and repo_version in versions:
+      # If repo version has build number and is found in the list of versions, return it as the matching version
+      Logger.info("Best Fit Version: Resolved from repo version with valid build number: {0}".format(repo_version))
+      return repo_version
+
+    # Get version without build number
+    normalized_repo_version = repo_version.split('-')[0]
+
+    # Find all versions that match the normalized repo version
+    match_versions = filter(lambda x: x.startswith(normalized_repo_version), versions)
+    if match_versions:
+
+      if len(match_versions) == 1:
+        # Resolved without conflicts
+        Logger.info("Best Fit Version: Resolved from normalized repo version without conflicts: {0}".format(match_versions[0]))
+        return match_versions[0]
+
+      # Resolve conflicts using REPO_VERSION_HISTORY_FILE
+      history_version = read_actual_version_from_history_file(normalized_repo_version)
+
+      # Validate history version retrieved is valid
+      if history_version in match_versions:
+        Logger.info("Best Fit Version: Resolved from normalized repo version using {0}: {1}".format(REPO_VERSION_HISTORY_FILE, history_version))
+        return history_version
+
+    # No matching version
+    return None
+
 
   def install_packages(self, package_list):
     """
@@ -245,7 +318,10 @@ class InstallPackages(Script):
             Package(package, action="remove")
     # Compute the actual version in order to save it in structured out
     try:
-      self.compute_actual_version()
+      if ret_code == 0:
+         self.compute_actual_version()
+      else:
+        self.check_partial_install()
     except Fail, err:
       ret_code = 1
       Logger.logger.exception("Failure while computing actual version. Error: {0}".format(str(err)))
@@ -297,7 +373,7 @@ class InstallPackages(Script):
 
   def abort_handler(self, signum, frame):
     Logger.error("Caught signal {0}, will handle it gracefully. Compute the actual version if possible before exiting.".format(signum))
-    self.compute_actual_version()
+    self.check_partial_install()
 
 
 if __name__ == "__main__":

ambari-server/src/test/python/custom_actions/TestInstallPackages.py

@@ -71,7 +71,10 @@ class TestInstallPackages(RMFTestCase):
                             read_actual_version_from_history_file_mock,
                             hdp_versions_mock,
                             put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock):
-    read_actual_version_from_history_file_mock.return_value = VERSION_STUB
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value=[]
     self.executeScript("scripts/install_packages.py",
@@ -126,7 +129,10 @@ class TestInstallPackages(RMFTestCase):
                             read_actual_version_from_history_file_mock,
                             hdp_versions_mock, put_structured_out_mock, allInstalledPackages_mock, list_ambari_managed_repos_mock, is_suse_family_mock):
     is_suse_family_mock = True
-    read_actual_version_from_history_file_mock.return_value = VERSION_STUB
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value=[]
     self.executeScript("scripts/install_packages.py",
@@ -183,7 +189,10 @@ class TestInstallPackages(RMFTestCase):
                                  hdp_versions_mock,
                                  allInstalledPackages_mock, put_structured_out_mock,
                                  is_redhat_family_mock, list_ambari_managed_repos_mock):
-    read_actual_version_from_history_file_mock.return_value = VERSION_STUB
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value=["HDP-UTILS-2.2.0.1-885"]
     is_redhat_family_mock.return_value = True
@@ -274,7 +283,6 @@ class TestInstallPackages(RMFTestCase):
     self.assertTrue(put_structured_out_mock.called)
     self.assertEquals(put_structured_out_mock.call_args[0][0],
                       {'stack_id': 'HDP-2.2',
-                      'actual_version': VERSION_STUB,
                       'installed_repository_version': VERSION_STUB,
                       'ambari_repositories': [],
                       'package_installation_result': 'FAIL'})
@@ -313,6 +321,10 @@ class TestInstallPackages(RMFTestCase):
                                hdp_versions_mock,
                                allInstalledPackages_mock, put_structured_out_mock,
                                package_mock, is_suse_family_mock):
+    hdp_versions_mock.side_effect = [
+      [],  # before installation attempt
+      [VERSION_STUB]
+    ]
     read_actual_version_from_history_file_mock.return_value = VERSION_STUB
     allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages)
     is_suse_family_mock.return_value = True
@@ -562,17 +574,21 @@ class TestInstallPackages(RMFTestCase):
 
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value = []
-    self.executeScript("scripts/install_packages.py",
+    try:
+      self.executeScript("scripts/install_packages.py",
                        classname="InstallPackages",
                        command="actionexecute",
                        config_dict=command_json,
                        target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
                        os_type=('Redhat', '6.4', 'Final'),
                        )
+      self.fail("Should throw exception")
+    except Fail:
+      pass  # Expected
 
     self.assertTrue(put_structured_out_mock.called)
     self.assertEquals(put_structured_out_mock.call_args[0][0],
-                      {'package_installation_result': 'SUCCESS',
+                      {'package_installation_result': 'FAIL',
                        'stack_id': u'HDP-2.2',
                        'installed_repository_version': VERSION_STUB,
                        'actual_version': VERSION_STUB,
@@ -806,17 +822,21 @@ class TestInstallPackages(RMFTestCase):
 
     allInstalledPackages_mock.side_effect = TestInstallPackages._add_packages
     list_ambari_managed_repos_mock.return_value = []
-    self.executeScript("scripts/install_packages.py",
+    try:
+      self.executeScript("scripts/install_packages.py",
                        classname="InstallPackages",
                        command="actionexecute",
                        config_dict=command_json,
                        target=RMFTestCase.TARGET_CUSTOM_ACTIONS,
                        os_type=('Redhat', '6.4', 'Final'),
                        )
+      self.fail("Should throw exception")
+    except Fail:
+      pass  # Expected
 
     self.assertTrue(put_structured_out_mock.called)
     self.assertEquals(put_structured_out_mock.call_args[0][0],
-                      {'package_installation_result': 'SUCCESS',
+                      {'package_installation_result': 'FAIL',
                        'stack_id': u'HDP-2.2',
                        'installed_repository_version': VERSION_STUB,
                        'actual_version': VERSION_STUB,