AMBARI-9096. Cluster create fails when "hive.server2.transport.mode" property is set to "http" (dlysnichenko)

ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py

@@ -22,7 +22,8 @@ from resource_management.core.resources import Execute
 from resource_management.libraries.functions import format
 import socket
 
-def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = 'ambari-qa'):
+def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = 'ambari-qa',
+                           transport_mode = "binary"):
   """
   Hive thrift SASL port check
   """
@@ -36,7 +37,7 @@ def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kini
   else:
     url = format("jdbc:hive2://{address}:{port}")
 
-  if hive_auth != "NOSASL":
+  if hive_auth != "NOSASL" and transport_mode != "http":
     cmd = format("! beeline -u '{url}' -e '' ") + "2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'"
     Execute(cmd,
             user=smokeuser,

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_hive_thrift_port.py

@@ -31,6 +31,8 @@ OK_MESSAGE = "TCP OK - %.4f response on port %s"
 CRITICAL_MESSAGE = "Connection failed on host {0}:{1}"
 
 HIVE_SERVER_THRIFT_PORT_KEY = '{{hive-site/hive.server2.thrift.port}}'
+HIVE_SERVER_THRIFT_HTTP_PORT_KEY = '{{hive-site/hive.server2.thrift.http.port}}'
+HIVE_SERVER_TRANSPORT_MODE_KEY = '{{hive-site/hive.server2.transport.mode}}'
 SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
 HIVE_SERVER2_AUTHENTICATION_KEY = '{{hive-site/hive.server2.authentication}}'
 HIVE_SERVER_PRINCIPAL_KEY = '{{hive-site/hive.server2.authentication.kerberos.principal}}'
@@ -41,6 +43,7 @@ PERCENT_WARNING = 200
 PERCENT_CRITICAL = 200
 
 THRIFT_PORT_DEFAULT = 10000
+HIVE_SERVER_TRANSPORT_MODE_DEFAULT = 'binary'
 HIVE_SERVER_PRINCIPAL_DEFAULT = 'hive/_HOST@EXAMPLE.COM'
 HIVE_SERVER2_AUTHENTICATION_DEFAULT = 'NOSASL'
 SMOKEUSER_KEYTAB_DEFAULT = '/etc/security/keytabs/smokeuser.headless.keytab'
@@ -51,7 +54,8 @@ def get_tokens():
   Returns a tuple of tokens in the format {{site/property}} that will be used
   to build the dictionary passed into execute
   """
-  return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY,SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY)
+  return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY,
+          SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY,HIVE_SERVER_THRIFT_HTTP_PORT_KEY,HIVE_SERVER_TRANSPORT_MODE_KEY)
 
 
 def execute(parameters=None, host_name=None):
@@ -66,9 +70,15 @@ def execute(parameters=None, host_name=None):
   if parameters is None:
     return (('UNKNOWN', ['There were no parameters supplied to the script.']))
 
-  thrift_port = THRIFT_PORT_DEFAULT
-  if HIVE_SERVER_THRIFT_PORT_KEY in parameters:
-    thrift_port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY])
+  transport_mode = HIVE_SERVER_TRANSPORT_MODE_DEFAULT
+  if HIVE_SERVER_TRANSPORT_MODE_KEY in parameters:
+    transport_mode = parameters[HIVE_SERVER_TRANSPORT_MODE_KEY]
+
+  port = THRIFT_PORT_DEFAULT
+  if transport_mode.lower() == 'binary' and HIVE_SERVER_THRIFT_PORT_KEY in parameters:
+    port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY])
+  elif  transport_mode.lower() == 'http' and HIVE_SERVER_THRIFT_HTTP_PORT_KEY in parameters:
+    port = int(parameters[HIVE_SERVER_THRIFT_HTTP_PORT_KEY])
 
   security_enabled = False
   if SECURITY_ENABLED_KEY in parameters:
@@ -103,8 +113,8 @@ def execute(parameters=None, host_name=None):
 
     start_time = time.time()
     try:
-      hive_check.check_thrift_port_sasl(host_name, thrift_port, hive_server2_authentication,
-                                        hive_server_principal, kinitcmd, smokeuser)
+      hive_check.check_thrift_port_sasl(host_name, port, hive_server2_authentication,
+                                        hive_server_principal, kinitcmd, smokeuser, transport_mode = transport_mode)
       is_thrift_port_ok = True
     except:
       is_thrift_port_ok = False
@@ -112,13 +122,13 @@ def execute(parameters=None, host_name=None):
     if is_thrift_port_ok == True:
       result_code = 'OK'
       total_time = time.time() - start_time
-      label = OK_MESSAGE % (total_time, thrift_port)
+      label = OK_MESSAGE % (total_time, port)
     else:
       result_code = 'CRITICAL'
-      label = CRITICAL_MESSAGE.format(host_name,thrift_port)
+      label = CRITICAL_MESSAGE.format(host_name,port)
 
   except Exception, e:
     label = str(e)
     result_code = 'UNKNOWN'
 
-  return ((result_code, [label]))
+  return ((result_code, [label]))

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_service.py

@@ -82,7 +82,8 @@ def hive_service(name, action='start', rolling_restart=False):
       while time.time() < end_time:
         try:
           check_thrift_port_sasl(address, port, params.hive_server2_authentication,
-                                 params.hive_server_principal, kinitcmd, params.smokeuser)
+                                 params.hive_server_principal, kinitcmd, params.smokeuser,
+                                 transport_mode=params.hive_transport_mode)
           is_service_socket_valid = True
           break
         except Exception, e:

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params.py

@@ -135,7 +135,11 @@ hive_var_lib = '/var/lib/hive'
 ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 hive_server_host = config['clusterHostInfo']['hive_server_host'][0]
 hive_server_hosts = config['clusterHostInfo']['hive_server_host']
-hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
+hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode']
+if hive_transport_mode.lower() == "http":
+  hive_server_port = config['configurations']['hive-site']['hive.server2.thrift.http.port']
+else:
+  hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
 hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}")
 hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
 hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication']

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/service_check.py

@@ -42,7 +42,8 @@ class HiveServiceCheck(Script):
     for address in address_list:
       try:
         check_thrift_port_sasl(address, port, params.hive_server2_authentication,
-                               params.hive_server_principal, kinitcmd, params.smokeuser)
+                               params.hive_server_principal, kinitcmd, params.smokeuser,
+                               transport_mode=params.hive_transport_mode)
         print "Successfully connected to %s on port %s" % (address, port)
         workable_server_available = True
       except:

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -370,6 +370,7 @@
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
             "hive.server2.enable.doAs": "true",
             "hive.server2.authentication": "NOSASL",
+            "hive.server2.transport.mode": "binary",
             "hive.optimize.mapjoin.mapreduce": "true"
         }, 
         "yarn-site": {

ambari-server/src/test/python/stacks/2.0.6/configs/default_client.json

@@ -364,7 +364,8 @@
             "ambari.hive.db.schema.name": "hive", 
             "hive.metastore.execute.setugi": "true", 
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
-            "hive.server2.enable.doAs": "true", 
+            "hive.server2.enable.doAs": "true",
+            "hive.server2.transport.mode": "binary",
             "hive.optimize.mapjoin.mapreduce": "true"
         }, 
         "yarn-site": {

ambari-server/src/test/python/stacks/2.0.6/configs/secured.json

@@ -407,7 +407,8 @@
             "hive.metastore.execute.setugi": "true", 
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
             "hive.server2.enable.doAs": "true", 
-            "hive.optimize.mapjoin.mapreduce": "true", 
+            "hive.optimize.mapjoin.mapreduce": "true",
+            "hive.server2.transport.mode": "binary",
             "hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
         }, 
         "yarn-site": {

ambari-server/src/test/python/stacks/2.0.6/configs/secured_client.json

@@ -405,7 +405,8 @@
             "hive.metastore.execute.setugi": "true", 
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
             "hive.server2.enable.doAs": "true", 
-            "hive.optimize.mapjoin.mapreduce": "true", 
+            "hive.optimize.mapjoin.mapreduce": "true",
+            "hive.server2.transport.mode": "binary",
             "hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
         }, 
         "yarn-site": {

ambari-server/src/test/python/stacks/2.1/configs/default.json

@@ -479,7 +479,8 @@
             "ambari.hive.db.schema.name": "hive", 
             "hive.metastore.execute.setugi": "true", 
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
-            "hive.server2.enable.doAs": "true", 
+            "hive.server2.enable.doAs": "true",
+            "hive.server2.transport.mode": "binary",
             "hive.optimize.mapjoin.mapreduce": "true"
         }, 
         "yarn-site": {

ambari-server/src/test/python/stacks/2.1/configs/secured.json

@@ -451,7 +451,8 @@
             "hive.metastore.execute.setugi": "true", 
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
             "hive.server2.enable.doAs": "true", 
-            "hive.optimize.mapjoin.mapreduce": "true", 
+            "hive.optimize.mapjoin.mapreduce": "true",
+            "hive.server2.transport.mode": "binary",
             "hive.server2.authentication.kerberos.keytab": "/etc/security/keytabs/hive.service.keytab"
         }, 
         "yarn-site": {