AMBARI-11667. Start NN in HA, Mapreduce, Oozie and some other are failing with HTTP SPNEGO configured.(vbrodetskyi)

ambari-common/src/main/python/resource_management/libraries/functions/jmx.py

@@ -19,13 +19,26 @@ limitations under the License.
 '''
 import urllib2
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
+from resource_management.core import shell
+from resource_management.core.logger import Logger
 
-def get_value_from_jmx(qry, property):
+def get_value_from_jmx(qry, property, security_enabled, run_user, is_https_enabled):
   try:
-    response = urllib2.urlopen(qry)
-    data = response.read()
+    if security_enabled:
+      cmd = ['curl', '--negotiate', '-u', ':']
+    else:
+      cmd = ['curl']
+
+    if is_https_enabled:
+      cmd.append("-k")
+
+    cmd.append(qry)
+
+    _, data = shell.checked_call(cmd, user=run_user, quiet=False)
+
     if data:
       data_dict = json.loads(data)
       return data_dict["beans"][0][property]
   except:
+    Logger.exception("Getting jmx metrics from NN failed. URL: " + str(qry))
     return None

ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py

@@ -31,7 +31,7 @@ NAMENODE_HTTP_FRAGMENT = 'dfs.namenode.http-address.{0}.{1}'
 NAMENODE_HTTPS_FRAGMENT = 'dfs.namenode.https-address.{0}.{1}'
 JMX_URI_FRAGMENT = "http://{0}/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus"
   
-def get_namenode_states(hdfs_site):
+def get_namenode_states(hdfs_site, security_enabled, run_user):
   """
   return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
   """
@@ -59,7 +59,7 @@ def get_namenode_states(hdfs_site):
       value = str(hdfs_site[key])
 
       jmx_uri = JMX_URI_FRAGMENT.format(value)
-      state = get_value_from_jmx(jmx_uri,'State')
+      state = get_value_from_jmx(jmx_uri, 'State', security_enabled, run_user, is_https_enabled)
       
       if state == HDFS_NN_STATE_ACTIVE:
         active_namenodes.append((nn_unique_id, value))
@@ -86,17 +86,17 @@ def is_ha_enabled(hdfs_site):
       
   return False
 
-def get_active_namenode(hdfs_site):
+def get_active_namenode(hdfs_site, security_enabled, run_user):
   """
   return format is nn_unique_id and it's address ('nn1', 'hdfs://hostname1:port1')
   """
-  active_namenodes = get_namenode_states(hdfs_site)[0]
+  active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0]
   if active_namenodes:
     return active_namenodes[0]
   else:
     return UnknownConfiguration('fs_root')
   
-def get_property_for_active_namenode(hdfs_site, property_name):
+def get_property_for_active_namenode(hdfs_site, property_name, security_enabled, run_user):
   """
   For dfs.namenode.rpc-address:
     - In non-ha mode it will return hdfs_site[dfs.namenode.rpc-address]
@@ -104,7 +104,7 @@ def get_property_for_active_namenode(hdfs_site, property_name):
   """
   if is_ha_enabled(hdfs_site):
     name_service = hdfs_site['dfs.nameservices']
-    active_namenodes = get_namenode_states(hdfs_site)[0]
+    active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0]
     
     if not len(active_namenodes):
       raise Fail("There is no active namenodes.")
@@ -114,4 +114,4 @@ def get_property_for_active_namenode(hdfs_site, property_name):
     return hdfs_site[format("{property_name}.{name_service}.{active_namenode_id}")]
   else:
     return hdfs_site[property_name]
-  
+  

ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py

@@ -120,8 +120,10 @@ class HdfsResourceJar:
 
 class WebHDFSUtil:
   def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None):
-    https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address')
-    http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address')
+    https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address',
+                                                                          security_enabled, run_user)
+    http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address',
+                                                                         security_enabled, run_user)
     self.is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty(hdfs_site['dfs.https.enable']) else False
     
     address = https_nn_address if self.is_https_enabled else http_nn_address

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/namenode_ha_state.py

@@ -81,7 +81,7 @@ class NamenodeHAState:
           raise Exception("Could not retrieve hostname from address " + actual_value)
 
         jmx_uri = jmx_uri_fragment.format(actual_value)
-        state = get_value_from_jmx(jmx_uri, "State")
+        state = get_value_from_jmx(jmx_uri, "State", params.security_enabled, params.hdfs_user, params.is_https_enabled)
 
         if not state:
           raise Exception("Could not retrieve Namenode state from URL " + jmx_uri)
@@ -175,4 +175,4 @@ class NamenodeHAState:
     """
     active_hosts = self.namenode_state_to_hostnames[NAMENODE_STATE.ACTIVE] if NAMENODE_STATE.ACTIVE in self.namenode_state_to_hostnames else []
     standby_hosts = self.namenode_state_to_hostnames[NAMENODE_STATE.STANDBY] if NAMENODE_STATE.STANDBY in self.namenode_state_to_hostnames else []
-    return len(active_hosts) == 1 and len(standby_hosts) == 1
+    return len(active_hosts) == 1 and len(standby_hosts) == 1

ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/params_linux.py

@@ -37,6 +37,7 @@ from resource_management.libraries.resources.hdfs_resource import HdfsResource
 
 from resource_management.libraries.functions.format_jvm_option import format_jvm_option
 from resource_management.libraries.functions.get_lzo_packages import get_lzo_packages
+from resource_management.libraries.functions.is_empty import is_empty
 
 
 config = Script.get_config()
@@ -384,6 +385,9 @@ policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_
 jdk_location = config['hostLevelParams']['jdk_location']
 java_share_dir = '/usr/share/java'
 
+is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enabled'] if \
+  not is_empty(config['configurations']['hdfs-site']['dfs.https.enabled']) else False
+
 if has_ranger_admin:
   enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes')
   xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])

ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py

@@ -283,7 +283,7 @@ class TestJournalnode(RMFTestCase):
     namenode_status_standby = open(namenode_status_standby_file, 'r').read()
 
     url_stream_mock = MagicMock()
-    url_stream_mock.read.side_effect = [namenode_status_active, namenode_status_standby] + (num_journalnodes * [namenode_jmx, journalnode_jmx])
+    url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx])
 
     urlopen_mock.return_value = url_stream_mock
 
@@ -291,6 +291,7 @@ class TestJournalnode(RMFTestCase):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
       classname = "JournalNode", command = "post_rolling_restart",
       config_file = "journalnode-upgrade.json",
+      checked_call_mocks = [(0, str(namenode_status_active)), (0, str(namenode_status_standby))],
       hdp_stack_version = self.UPGRADE_STACK_VERSION,
       target = RMFTestCase.TARGET_COMMON_SERVICES )
 
@@ -299,7 +300,7 @@ class TestJournalnode(RMFTestCase):
     urlopen_mock.assert_called_with("http://c6407.ambari.apache.org:8480/jmx")
 
     url_stream_mock.reset_mock()
-    url_stream_mock.read.side_effect = [namenode_status_active, namenode_status_standby] + (num_journalnodes * [namenode_jmx, journalnode_jmx])
+    url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx])
 
     urlopen_mock.return_value = url_stream_mock
 
@@ -307,6 +308,7 @@ class TestJournalnode(RMFTestCase):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
       classname = "JournalNode", command = "post_rolling_restart",
       config_file = "journalnode-upgrade-hdfs-secure.json",
+      checked_call_mocks = [(0, str(namenode_status_active)), (0, str(namenode_status_standby))],
       hdp_stack_version = self.UPGRADE_STACK_VERSION,
       target = RMFTestCase.TARGET_COMMON_SERVICES )