AMBARI-11743. NameNode is forced to leave safemode, which causes HBMaster master to crash if done too quickly (alejandro)

ambari-common/src/main/python/resource_management/libraries/functions/copy_tarball.py

@@ -73,14 +73,15 @@ def _get_single_version_from_hdp_select():
     code, stdoutdata = shell.call(get_hdp_versions_cmd, logoutput=True)
     with open(tmp_file, 'r+') as file:
       out = file.read()
-  except:
-    Logger.error("Could not parse output of {0}".format(str(tmp_file)))
+  except Exception, e:
+    Logger.logger.exception("Could not parse output of {0}. Error: {1}".format(str(tmp_file), str(e)))
   finally:
     try:
       if os.path.exists(tmp_file):
         os.remove(tmp_file)
-    except:
-      pass
+    except Exception, e:
+      Logger.logger.exception("Could not remove file {0}. Error: {1}".format(str(tmp_file), str(e)))
+
   if code != 0 or out is None or out == "":
     Logger.error("Could not verify HDP version by calling '{0}'. Return Code: {1}, Output: {2}.".format(get_hdp_versions_cmd, str(code), str(out)))
     return None

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/hdfs_namenode.py

@@ -89,61 +89,53 @@ def namenode(action=None, do_format=True, rolling_restart=False, env=None):
       Execute(format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}"),
               user = params.hdfs_user)
 
-    is_namenode_safe_mode_off = format("hadoop dfsadmin -fs {namenode_address} -safemode get | grep 'Safe mode is OFF'")
+    is_namenode_safe_mode_off = format("hdfs dfsadmin -fs {namenode_address} -safemode get | grep 'Safe mode is OFF'")
     if params.dfs_ha_enabled:
       is_active_namenode_cmd = as_user(format("hdfs --config {hadoop_conf_dir} haadmin -getServiceState {namenode_id} | grep active"), params.hdfs_user, env={'PATH':params.hadoop_bin_dir})
     else:
       is_active_namenode_cmd = None
 
-    # During normal operations, if HA is enabled and it is in standby, then stay in current state, otherwise, leave safemode.
-    # During Rolling Upgrade, both namenodes must leave safemode.
+    # During normal operations, if HA is enabled and it is in standby, then no need to check safemode staus.
+    # During Rolling Upgrade, both namenodes must eventually leave safemode, and Ambari can wait for this.
 
     # ___Scenario_________|_Expected safemode state___|_Wait for safemode OFF____|
     # 1 (HA and active)   | ON -> OFF                 | Yes                      |
     # 2 (HA and standby)  | no change (yes during RU) | no check (yes during RU) |
     # 3 (no-HA)           | ON -> OFF                 | Yes                      |
-    leave_safe_mode = False
+    check_for_safemode_off = False
     msg = ""
     if params.dfs_ha_enabled:
       code, out = shell.call(is_active_namenode_cmd, logoutput=True) # If active NN, code will be 0
       if code == 0: # active
-        leave_safe_mode = True
-        msg = "Must leave safemode since High Availability is enabled and this is the Active NameNode."
+        check_for_safemode_off = True
+        msg = "Must wait to leave safemode since High Availability is enabled and this is the Active NameNode."
       elif rolling_restart:
-        leave_safe_mode = True
-        msg = "Must leave safemode since High Availability is enabled during a Rolling Upgrade"
+        check_for_safemode_off = True
+        msg = "Must wait to leave safemode since High Availability is enabled during a Rolling Upgrade"
     else:
-      msg = "Must leave safemode since High Availability is not enabled."
-      leave_safe_mode = True
+      msg = "Must wait to leave safemode since High Availability is not enabled."
+      check_for_safemode_off = True
 
     if not msg:
       msg = "Will remain in the current safemode state."
     Logger.info(msg)
 
-    if leave_safe_mode:
-      # First check if Namenode is not in 'safemode OFF' (equivalent to safemode ON), if so, then leave it
+    if check_for_safemode_off:
+      # First check if Namenode is not in 'safemode OFF' (equivalent to safemode ON). If safemode is OFF, no change.
+      # If safemode is ON, first wait for NameNode to leave safemode on its own (if that doesn't happen within 30 seconds, then
+      # force NameNode to leave safemode).
       Logger.info("Checking the NameNode safemode status since may need to transition from ON to OFF.")
-      code, out = shell.call(is_namenode_safe_mode_off, user=params.hdfs_user)
-      if code != 0:
-        Logger.info("Will need to leave safemode, state should be OFF.")
-        leave_safe_mode_cmd = format("hdfs --config {hadoop_conf_dir} dfsadmin -fs {namenode_address} -safemode leave")
-        Execute(leave_safe_mode_cmd,
-                tries=10,
-                try_sleep=10,
-                user=params.hdfs_user,
-                path=[params.hadoop_bin_dir],
-        )
 
-        Logger.info("Checking if safemode state is now OFF.")
-        # Verify if Namenode should be in safemode OFF
+      try:
+        # Wait up to 30 mins
         Execute(is_namenode_safe_mode_off,
-                tries=40,
+                tries=180,
                 try_sleep=10,
-                path=[params.hadoop_bin_dir],
-                user=params.hdfs_user
+                user=params.hdfs_user,
+                logoutput=True
         )
-        pass
-      pass
+      except Fail:
+        Logger.error("NameNode is still in safemode, please be careful with commands that need safemode OFF.")
 
     # Always run this on non-HA, or active NameNode during HA.
     create_hdfs_directories(is_active_namenode_cmd)

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_namenode.py

@@ -50,7 +50,7 @@ class TestNamenode(RMFTestCase):
                        config_file = "altfs_plus_hdfs.json",
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
-                       call_mocks = [(5,"")],
+                       call_mocks = [(0,"")],
     )
     self.assert_configure_default()
     self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l  | grep -q ^0$',)
@@ -87,18 +87,12 @@ class TestNamenode(RMFTestCase):
                               environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
                               not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
                               )
-    self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf dfsadmin -fs hdfs://c6405.ambari.apache.org:8020 -safemode leave',
-                              path = ['/usr/bin'],
-                              tries = 10,
-                              try_sleep = 10,
-                              user = 'hdfs',
-                              )
-    self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6405.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'",
-                              path = ['/usr/bin'],
-                              tries = 40,
-                              user = 'hdfs',
-                              try_sleep = 10,
-                              )
+    self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://c6405.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'",
+        tries=180,
+        try_sleep=10,
+        user="hdfs",
+        logoutput=True
+    )
     self.assertResourceCalled('HdfsResource', '/tmp',
         security_enabled = False,
         only_if=None,
@@ -166,7 +160,7 @@ class TestNamenode(RMFTestCase):
                        config_file = "default.json",
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
-                       call_mocks = [(5,"")],
+                       call_mocks = [(0,"")],
     )
     self.assert_configure_default()
     self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l  | grep -q ^0$',)
@@ -203,17 +197,11 @@ class TestNamenode(RMFTestCase):
         environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
         not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
     )
-    self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode leave',
-        path = ['/usr/bin'],
-        tries = 10,
-        try_sleep = 10,
-        user = 'hdfs',
-    )
-    self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'",
-        path = ['/usr/bin'],
-        tries = 40,
-        user = 'hdfs',
-        try_sleep = 10,
+    self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'",
+        tries=180,
+        try_sleep=10,
+        user="hdfs",
+        logoutput=True
     )
     self.assertResourceCalled('HdfsResource', '/tmp',
         security_enabled = False,
@@ -302,7 +290,7 @@ class TestNamenode(RMFTestCase):
                        config_file = "secured.json",
                        hdp_stack_version = self.STACK_VERSION,
                        target = RMFTestCase.TARGET_COMMON_SERVICES,
-                       call_mocks = [(5,"")],
+                       call_mocks = [(0,"")],
     )
     self.assert_configure_secured()
     self.assertResourceCalled('Execute', 'ls /hadoop/hdfs/namenode | wc -l  | grep -q ^0$',)
@@ -342,17 +330,11 @@ class TestNamenode(RMFTestCase):
     self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',
                               user='hdfs',
                               )
-    self.assertResourceCalled('Execute', 'hdfs --config /etc/hadoop/conf dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode leave',
-        path = ['/usr/bin'],
-        tries = 10,
-        try_sleep = 10,
-        user = 'hdfs',
-    )
-    self.assertResourceCalled('Execute', "hadoop dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'",
-        path = ['/usr/bin'],
-        tries = 40,
-        user = 'hdfs',
-        try_sleep = 10,
+    self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://c6401.ambari.apache.org:8020 -safemode get | grep 'Safe mode is OFF'",
+        tries=180,
+        try_sleep=10,
+        user="hdfs",
+        logoutput=True
     )
     self.assertResourceCalled('HdfsResource', '/tmp',
         security_enabled = True,
@@ -448,6 +430,12 @@ class TestNamenode(RMFTestCase):
         environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
         not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
     )
+    self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'",
+        tries=180,
+        try_sleep=10,
+        user="hdfs",
+        logoutput=True
+    )
     self.assertResourceCalled('HdfsResource', '/tmp',
         security_enabled = False,
         only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export  PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'",
@@ -533,6 +521,12 @@ class TestNamenode(RMFTestCase):
     self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab hdfs',
         user = 'hdfs',
     )
+    self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'",
+        tries=180,
+        try_sleep=10,
+        user="hdfs",
+        logoutput=True
+    )
     self.assertResourceCalled('HdfsResource', '/tmp',
         security_enabled = True,
         only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export  PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'",
@@ -627,6 +621,12 @@ class TestNamenode(RMFTestCase):
                               environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
                               not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
                               )
+    self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'",
+        tries=180,
+        try_sleep=10,
+        user="hdfs",
+        logoutput=True
+    )
     self.assertResourceCalled('HdfsResource', '/tmp',
         security_enabled = False,
         only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export  PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn1 | grep active'",
@@ -677,7 +677,7 @@ class TestNamenode(RMFTestCase):
   # tests namenode start command when NameNode HA is enabled, and
   # the HA cluster is started initially, rather than using the UI Wizard
   # this test verifies the startup of a "standby" namenode
-  @patch.object(shell, "call", new=MagicMock(return_value=(5,"")))
+  @patch.object(shell, "call", new=MagicMock(return_value=(0,"")))
   def test_start_ha_bootstrap_standby_from_blueprint(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/namenode.py",
                        classname = "NameNode",
@@ -718,7 +718,12 @@ class TestNamenode(RMFTestCase):
                               environment = {'HADOOP_LIBEXEC_DIR': '/usr/lib/hadoop/libexec'},
                               not_if = 'ls /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid >/dev/null 2>&1 && ps -p `cat /var/run/hadoop/hdfs/hadoop-hdfs-namenode.pid` >/dev/null 2>&1',
                               )
-
+    self.assertResourceCalled('Execute', "hdfs dfsadmin -fs hdfs://ns1 -safemode get | grep 'Safe mode is OFF'",
+        tries=180,
+        try_sleep=10,
+        user="hdfs",
+        logoutput=True
+    )
     self.assertResourceCalled('HdfsResource', '/tmp',
         security_enabled = False,
         only_if = "ambari-sudo.sh su hdfs -l -s /bin/bash -c 'export  PATH=/bin:/usr/bin ; hdfs --config /etc/hadoop/conf haadmin -getServiceState nn2 | grep active'",

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -29,16 +29,17 @@ from resource_management.core import shell
 from resource_management.libraries.script.script import Script
 from resource_management.libraries.functions import copy_tarball
 from resource_management.libraries import functions
-
+from resource_management.core.logger import Logger
 
 @patch.object(functions, "get_hdp_version", new = MagicMock(return_value="2.0.0.0-1234"))
 @patch("resource_management.libraries.functions.check_thrift_port_sasl", new=MagicMock())
-#@patch("atlas_plugin_utils.configure_for_plugin", new=MagicMock())
 class TestHiveServer(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "HIVE/0.12.0.2.0/package"
   STACK_VERSION = "2.0.6"
   UPGRADE_STACK_VERSION = "2.2"
 
+  def setUp(self):
+    Logger.logger = MagicMock()
 
   @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False))
@@ -54,9 +55,11 @@ class TestHiveServer(RMFTestCase):
     self.assert_configure_default()
     self.assertNoMoreResources()
 
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   @patch("socket.socket")
   @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False))
-  def test_start_default(self, socket_mock):
+  def test_start_default(self, socket_mock, copy_to_hfds_mock):
+    copy_to_hfds_mock.return_value = None
     s = socket_mock.return_value
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
                        classname="HiveServer",
@@ -116,8 +119,10 @@ class TestHiveServer(RMFTestCase):
     )
     self.assertNoMoreResources()
 
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False))
-  def test_start_default_alt_tmp(self):
+  def test_start_default_alt_tmp(self, copy_to_hfds_mock):
+    copy_to_hfds_mock.return_value = None
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
                        classname = "HiveServer",
                        command = "start",
@@ -144,9 +149,10 @@ class TestHiveServer(RMFTestCase):
     )
     self.assertNoMoreResources()
 
-
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False))
-  def test_start_default_alt_nn_ha_tmp(self):
+  def test_start_default_alt_nn_ha_tmp(self, copy_to_hfds_mock):
+    copy_to_hfds_mock.return_value = None
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
                        classname = "HiveServer",
                        command = "start",
@@ -211,11 +217,13 @@ class TestHiveServer(RMFTestCase):
     self.assert_configure_secured()
     self.assertNoMoreResources()
 
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
   @patch("hive_service.check_fs_root")
   @patch("socket.socket")
   @patch.object(Script, "is_hdp_stack_greater_or_equal", new = MagicMock(return_value=False))
-  def test_start_secured(self, socket_mock, check_fs_root_mock):
+  def test_start_secured(self, socket_mock, check_fs_root_mock, copy_to_hfds_mock):
     s = socket_mock.return_value
+    copy_to_hfds_mock.return_value = None
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server.py",
                        classname = "HiveServer",
@@ -302,6 +310,10 @@ class TestHiveServer(RMFTestCase):
         mode = 0755,
     )
 
+    if self._testMethodName == "test_socket_timeout":
+      # This test will not call any more resources.
+      return
+
     self.assertResourceCalled('HdfsResource', '/apps/hive/warehouse',
         security_enabled = False,
         hadoop_bin_dir = '/usr/bin',
@@ -630,10 +642,9 @@ class TestHiveServer(RMFTestCase):
                               cd_access='a',
     )
 
-  @patch("hive_service.check_fs_root")
   @patch("time.time")
   @patch("socket.socket")
-  def test_socket_timeout(self, socket_mock, time_mock, check_fs_root_mock):
+  def test_socket_timeout(self, socket_mock, time_mock):
     s = socket_mock.return_value
     s.connect = MagicMock()    
     s.connect.side_effect = socket.error("")

ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py

@@ -26,7 +26,9 @@ class TestJobHistoryServer(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
   STACK_VERSION = "2.2"
 
-  def test_configure_default(self):
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+  def test_configure_default(self, copy_to_hdfs_mock):
+    copy_to_hdfs_mock = True
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
                    classname = "JobHistoryServer",
                    command = "configure",
@@ -36,8 +38,10 @@ class TestJobHistoryServer(RMFTestCase):
     )
     self.assert_configure_default()
     self.assertNoMoreResources()
-    
-  def test_start_default(self):
+
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+  def test_start_default(self, copy_to_hdfs_mock):
+    copy_to_hdfs_mock.return_value = True
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
                    classname = "JobHistoryServer",
                    command = "start",
@@ -46,6 +50,18 @@ class TestJobHistoryServer(RMFTestCase):
                    target = RMFTestCase.TARGET_COMMON_SERVICES
     )
     self.assert_configure_default()
+    self.assertResourceCalled('HdfsResource', None,
+        security_enabled = False,
+        hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+        keytab = UnknownConfigurationMock(),
+        default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+        hdfs_site = {u'a': u'b'},
+        kinit_path_local = '/usr/bin/kinit',
+        principal_name = UnknownConfigurationMock(),
+        user = 'hdfs',
+        action = ['execute'],
+        hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+    )
     self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-history-server.sh',
         environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
         not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid` >/dev/null 2>&1',
@@ -81,7 +97,9 @@ class TestJobHistoryServer(RMFTestCase):
     self.assert_configure_secured()
     self.assertNoMoreResources()
 
-  def test_start_secured(self):
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+  def test_start_secured(self, copy_to_hdfs_mock):
+    copy_to_hdfs_mock.return_value = True
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
                    classname = "JobHistoryServer",
                    command = "start",
@@ -94,6 +112,19 @@ class TestJobHistoryServer(RMFTestCase):
         user = 'spark',
     )
 
+    self.assertResourceCalled('HdfsResource', None,
+        action=['execute'],
+        default_fs= UnknownConfigurationMock(),
+        hadoop_bin_dir='/usr/hdp/current/hadoop-client/bin',
+        hadoop_conf_dir='/usr/hdp/current/hadoop-client/conf',
+        hdfs_site=UnknownConfigurationMock(),
+        keytab=UnknownConfigurationMock(),
+        kinit_path_local='/usr/bin/kinit',
+        principal_name=UnknownConfigurationMock(),
+        security_enabled=True,
+        user=UnknownConfigurationMock()
+    )
+
     self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-history-server.sh',
         environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
         not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid` >/dev/null 2>&1',