AMBARI-12534 When NameNode HA is enabled and JMX info is not available, RM/HBase Master, and other masters cannot be started (dsen)

ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py

@@ -22,6 +22,8 @@ from resource_management.libraries.functions.is_empty import is_empty
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.jmx import get_value_from_jmx
 from resource_management.core.base import Fail
+from resource_management.core import shell
+
 __all__ = ["get_namenode_states", "get_active_namenode", "get_property_for_active_namenode"]
 
 HDFS_NN_STATE_ACTIVE = 'active'
@@ -63,14 +65,23 @@ def get_namenode_states(hdfs_site, security_enabled, run_user):
       jmx_uri = JMX_URI_FRAGMENT.format(protocol, value)
       
       state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled)
-      
+      # If JMX parsing failed
+      if not state:
+        check_service_cmd = "hdfs haadmin -getServiceState {0}".format(nn_unique_id)
+        code, out = shell.call(check_service_cmd, logoutput=True, user=run_user)
+        if code == 0 and out:
+          if HDFS_NN_STATE_STANDBY in out:
+            state = HDFS_NN_STATE_STANDBY
+          elif HDFS_NN_STATE_ACTIVE in out:
+            state = HDFS_NN_STATE_ACTIVE
+
       if state == HDFS_NN_STATE_ACTIVE:
         active_namenodes.append((nn_unique_id, value))
       elif state == HDFS_NN_STATE_STANDBY:
         standby_namenodes.append((nn_unique_id, value))
       else:
         unknown_namenodes.append((nn_unique_id, value))
-        
+
   return active_namenodes, standby_namenodes, unknown_namenodes
 
 def is_ha_enabled(hdfs_site):

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py

@@ -17,7 +17,7 @@ limitations under the License.
 
 """
 
-
+from resource_management.core import shell
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.jmx import get_value_from_jmx
@@ -83,6 +83,17 @@ class NamenodeHAState:
         jmx_uri = jmx_uri_fragment.format(actual_value)
         state = get_value_from_jmx(jmx_uri, "tag.HAState", params.security_enabled, params.hdfs_user, params.is_https_enabled)
 
+        # If JMX parsing failed
+        if not state:
+          run_user = default("/configurations/hadoop-env/hdfs_user", "hdfs")
+          check_service_cmd = "hdfs haadmin -getServiceState {0}".format(nn_unique_id)
+          code, out = shell.call(check_service_cmd, logoutput=True, user=run_user)
+          if code == 0 and out:
+            if NAMENODE_STATE.STANDBY in out:
+              state = NAMENODE_STATE.STANDBY
+            elif NAMENODE_STATE.ACTIVE in out:
+              state = NAMENODE_STATE.ACTIVE
+
         if not state:
           raise Exception("Could not retrieve Namenode state from URL " + jmx_uri)