AMBARI-11367. RU - Knox upgrade fails because topologies folder is not created, and admin.xml not copied (alejandro)

ambari-agent/src/main/python/ambari_agent/Controller.py

@@ -365,21 +365,23 @@ class Controller(threading.Thread):
 
   def registerAndHeartbeat(self):
     registerResponse = self.registerWithServer()
-    message = registerResponse['response']
-    logger.info("Registration response from %s was %s", self.serverHostname, message)
 
-    if self.isRegistered:
-      # Clearing command queue to stop executing "stale" commands
-      # after registration
-      logger.info('Resetting ActionQueue...')
-      self.actionQueue.reset()
+    if "response" in registerResponse:
+      message = registerResponse["response"]
+      logger.info("Registration response from %s was %s", self.serverHostname, message)
 
-      # Process callbacks
-      for callback in self.registration_listeners:
-        callback()
+      if self.isRegistered:
+        # Clearing command queue to stop executing "stale" commands
+        # after registration
+        logger.info('Resetting ActionQueue...')
+        self.actionQueue.reset()
 
-      time.sleep(self.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC)
-      self.heartbeatWithServer()
+        # Process callbacks
+        for callback in self.registration_listeners:
+          callback()
+
+        time.sleep(self.netutil.HEARTBEAT_IDDLE_INTERVAL_SEC)
+        self.heartbeatWithServer()
 
   def restartAgent(self):
     sys.exit(AGENT_AUTO_RESTART_EXIT_CODE)

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox.py

@@ -18,10 +18,20 @@ limitations under the License.
 """
 
 import os
-from resource_management import *
+
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.core.resources.service import ServiceConfig
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.resources.template_config import TemplateConfig
+from resource_management.core.resources.system import File, Execute, Directory
+from resource_management.core.shell import as_user
+from resource_management.core.source import InlineTemplate
+
 from ambari_commons import OSConst
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 
+from resource_management.core.logger import Logger
+
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
 def knox():
   import params
@@ -66,7 +76,7 @@ def knox():
 def knox():
     import params
 
-    directories = [params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir]
+    directories = [params.knox_data_dir, params.knox_logs_dir, params.knox_pid_dir, params.knox_conf_dir, os.path.join(params.knox_conf_dir, "topologies")]
     for directory in directories:
       Directory(directory,
                 owner = params.knox_user,

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py

@@ -17,22 +17,35 @@ limitations under the License.
 
 """
 
-from resource_management import *
+import os
+import tarfile
 
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions import Direction
 from resource_management.libraries.functions.security_commons import build_expectations, \
   cached_kinit_executor, validate_security_config_properties, get_params_from_filesystem, \
   FILE_TYPE_XML
-import sys
+from resource_management.core.resources.system import File, Execute, Directory
+from resource_management.core.resources.service import Service
+from resource_management.core.logger import Logger
+
+from ambari_commons import OSConst, OSCheck
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+if OSCheck.is_windows_family():
+  from resource_management.libraries.functions.windows_service_utils import check_windows_service_status
+
 import upgrade
-import os
 from knox import knox
 from knox_ldap import ldap
 from setup_ranger_knox import setup_ranger_knox
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
 
 class KnoxGateway(Script):
   def install(self, env):
@@ -98,10 +111,39 @@ class KnoxGatewayDefault(KnoxGateway):
     import params
     env.set_params(params)
     if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
-      upgrade.backup_data()
+
+      absolute_backup_dir = None
+      if params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE:
+        Logger.info("Backing up directories. Initial conf folder: %s" % os.path.realpath(params.knox_conf_dir))
+
+        # This will backup the contents of the conf directory into /tmp/knox-upgrade-backup/knox-conf-backup.tar
+        absolute_backup_dir = upgrade.backup_data()
+
+      # conf-select will change the symlink to the conf folder.
       conf_select.select(params.stack_name, "knox", params.version)
       hdp_select.select("knox-server", params.version)
 
+      # Extract the tar of the old conf folder into the new conf directory
+      if absolute_backup_dir is not None and params.upgrade_direction and params.upgrade_direction == Direction.UPGRADE:
+        conf_tar_source_path = os.path.join(absolute_backup_dir, upgrade.BACKUP_CONF_ARCHIVE)
+        if os.path.exists(conf_tar_source_path):
+          extract_dir = os.path.realpath(params.knox_conf_dir)
+          conf_tar_dest_path = os.path.join(extract_dir, upgrade.BACKUP_CONF_ARCHIVE)
+          Logger.info("Copying %s into %s file." % (upgrade.BACKUP_CONF_ARCHIVE, conf_tar_dest_path))
+          Execute("cp %s %s" % (conf_tar_source_path, conf_tar_dest_path))
+
+          tarball = None
+          try:
+            tarball = tarfile.open(conf_tar_source_path, "r")
+            Logger.info("Extracting %s into %s directory." % (upgrade.BACKUP_CONF_ARCHIVE, extract_dir))
+            tarball.extractall(extract_dir)
+
+            Logger.info("Deleting temporary tar at %s" % conf_tar_dest_path)
+            Execute("rm %s" % (conf_tar_dest_path))
+          finally:
+            if tarball:
+              tarball.close()
+
   def start(self, env, rolling_restart=False):
     import params
     env.set_params(params)

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py

@@ -33,6 +33,7 @@ config = Script.get_config()
 
 tmp_dir = Script.get_tmp_dir()
 stack_name = default("/hostLevelParams/stack_name", None)
+upgrade_direction = default("/commandParams/upgrade_direction", None)
 version = default("/commandParams/version", None)
 
 knox_master_secret_path = '/var/lib/knox/data/security/master'

ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py

@@ -32,7 +32,7 @@ BACKUP_CONF_ARCHIVE = "knox-conf-backup.tar"
 def backup_data():
   """
   Backs up the knox data as part of the upgrade process.
-  :return:
+  :return: Returns the path to the absolute backup directory.
   """
   Logger.info('Backing up Knox data directory before upgrade...')
   directoryMappings = _get_directory_mappings()
@@ -58,6 +58,7 @@ def backup_data():
     finally:
       if tarball:
         tarball.close()
+  return absolute_backup_dir
 
 def _get_directory_mappings():
   """
@@ -67,5 +68,6 @@ def _get_directory_mappings():
   """
   import params
 
-  return { params.knox_data_dir : BACKUP_DATA_ARCHIVE, params.knox_conf_dir : BACKUP_CONF_ARCHIVE }
+  return { params.knox_data_dir : BACKUP_DATA_ARCHIVE,
+           params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE} # the trailing "/" is important here so as to not include the "conf" folder itself
 

ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py

@@ -23,6 +23,26 @@ from stacks.utils.RMFTestCase import *
 from mock.mock import patch
 from mock.mock import MagicMock
 
+
+class TarfileFake:
+  """
+  Dummy class to mock the calls to tarfile module.
+  """
+  def __init__(self):
+    self.extractall_count = 0
+    self.add_count = 0
+    self.close_count = 0
+
+  def extractall(self, path=".", members=None):
+    self.extractall_count += 1
+
+  def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
+    self.add_count += 1
+
+  def close(self):
+    self.close_count += 1
+
+
 @patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
 class TestKnoxGateway(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "KNOX/0.5.0.2.2/package"
@@ -57,6 +77,11 @@ class TestKnoxGateway(RMFTestCase):
                               group = 'knox',
                               recursive = True
     )
+    self.assertResourceCalled('Directory', '/usr/hdp/current/knox-server/conf/topologies',
+                              owner = 'knox',
+                              group = 'knox',
+                              recursive = True
+    )
 
     self.assertResourceCalled('XmlConfig', 'gateway-site.xml',
                               owner = 'knox',
@@ -83,7 +108,8 @@ class TestKnoxGateway(RMFTestCase):
      '/var/lib/knox/data',
      '/var/log/knox',
      '/var/run/knox',
-     '/usr/hdp/current/knox-server/conf'),
+     '/usr/hdp/current/knox-server/conf',
+     '/usr/hdp/current/knox-server/conf/topologies'),
         sudo = True,
     )
     self.assertResourceCalled('Execute', '/usr/hdp/current/knox-server/bin/knoxcli.sh create-master --master sa',
@@ -216,7 +242,7 @@ class TestKnoxGateway(RMFTestCase):
   @patch("os.path.isdir")
   def test_pre_rolling_restart(self, isdir_mock, tarfile_open_mock):
     isdir_mock.return_value = True
-    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
     version = '2.2.1.0-3242'
@@ -233,18 +259,24 @@ class TestKnoxGateway(RMFTestCase):
 
     self.assertResourceCalled("Execute", "hdp-select set knox-server %s" % version)
 
+  @patch("os.remove")
+  @patch("os.path.exists")
   @patch("tarfile.open")
   @patch("os.path.isdir")
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, call_mock, isdir_mock, tarfile_open_mock):
+  def test_pre_rolling_restart_23(self, call_mock, isdir_mock, tarfile_open_mock, path_exists_mock, remove_mock):
     isdir_mock.return_value = True
-    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
     version = '2.3.0.0-1234'
     json_content['commandParams']['version'] = version
 
+    path_exists_mock.return_value = True
+    knox_conf_tarfile = TarfileFake()
+    tarfile_open_mock.return_value = knox_conf_tarfile
     mocks_dict = {}
+
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
                        classname = "KnoxGateway",
                        command = "pre_rolling_restart",
@@ -266,6 +298,9 @@ class TestKnoxGateway(RMFTestCase):
       "conf-select set-conf-dir --package knox --stack-version 2.3.0.0-1234 --conf-version 0",
        mocks_dict['call'].call_args_list[1][0][0])
 
+    self.assertEquals(1, knox_conf_tarfile.extractall_count)
+    self.assertTrue(2, knox_conf_tarfile.close_count)
+
   @patch("os.path.islink")
   @patch("os.path.realpath")
   @patch("os.unlink")
@@ -301,6 +336,11 @@ class TestKnoxGateway(RMFTestCase):
                               group = 'knox',
                               recursive = True
     )
+    self.assertResourceCalled('Directory', '/usr/hdp/current/knox-server/conf/topologies',
+                              owner = 'knox',
+                              group = 'knox',
+                              recursive = True
+    )
 
     self.assertResourceCalled('XmlConfig', 'gateway-site.xml',
                               owner = 'knox',
@@ -327,7 +367,7 @@ class TestKnoxGateway(RMFTestCase):
                                           '/var/lib/knox/data',
                                           '/var/log/knox',
                                           '/var/run/knox',
-                                          '/usr/hdp/current/knox-server/conf'),
+                                          '/usr/hdp/current/knox-server/conf', '/usr/hdp/current/knox-server/conf/topologies'),
                               sudo = True,
                               )
     self.assertResourceCalled('Execute', '/usr/hdp/current/knox-server/bin/knoxcli.sh create-master --master sa',