AMBARI-8523. ambari cluster HS2 health check causing errors in logs (aonishuk)

ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py

@@ -17,72 +17,39 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 '''
-import socket
 from resource_management.core.exceptions import Fail
+from resource_management.core.resources import Execute
+from resource_management.libraries.functions import format
+import socket
 
-def check_thrift_port_sasl(address, port, timeout = 5, security_enabled = False):
+def check_thrift_port_sasl(address, port, hive_auth = "NOSASL", key = None, kinitcmd = None, smokeuser = 'ambari-qa'):
   """
   Hive thrift SASL port check
   """
+  BEELINE_CHECK_TIMEOUT = 30
+
+  if kinitcmd:
+    url = format("jdbc:hive2://{address}:{port}/;principal={key}")
+    Execute(kinitcmd,
+            user=smokeuser
+    )
+  else:
+    url = format("jdbc:hive2://{address}:{port}")
+
+  if hive_auth != "NOSASL":
+    cmd = format("! beeline -u '{url}' -e '' ") + "2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'"
+    Execute(cmd,
+            user=smokeuser,
+            path=["/bin/", "/usr/bin/", "/usr/lib/hive/bin/", "/usr/sbin/"],
+            timeout=BEELINE_CHECK_TIMEOUT
+    )
+  else:
+    s = socket.socket()
+    s.settimeout(1)
+    try:
+      s.connect((address, port))
+    except socket.error, e:
+      raise
+    finally:
+      s.close()
 
-  #Authentification mechanism
-  mechanism = "PLAIN"
-  #Anonymous username
-  usr = "ANONYMOUS"
-  start_byte = 0x01 #START communication
-  ok_byte = 0x02 #OK
-  bad_byte = 0x03 #BAD
-  error_byte = 0x04 #ERROR
-  complete_byte = 0x05 #COMPLETE communication
-  
-  msg = bytearray()
-
-  msg.append(start_byte)
-  msg.append(0)
-  msg.append(0)
-  msg.append(0)
-  msg.append(len(mechanism))
-  for elem in mechanism:
-    msg.append(ord(elem))
-
-  msg.append(ok_byte)
-  msg.append(0)
-  msg.append(0)
-  msg.append(0)
-  msg.append(len(usr)*2+2)
-  
-  #Adding anonymous user name
-  msg.append(0)
-  for elem in usr:
-    msg.append(ord(elem))
-
-  #Adding anonymous user password
-  msg.append(0)
-  for elem in usr:
-    msg.append(ord(elem))
-
-  msg.append(complete_byte)
-  msg.append(0)
-  msg.append(0)
-  msg.append(0)
-  msg.append(0)
-
-  is_service_socket_valid = False
-  s = socket.socket()
-  s.settimeout(timeout)
-
-  try:
-    s.connect((address, port))
-    #Successfull connection, port check passed
-    is_service_socket_valid = True
-
-    # Try to send anonymous plain auth message to thrift to prevent errors in hive log
-    # Plain mechanism is not supported in security mode
-    if not security_enabled:
-      s.send(msg)
-  except socket.error, e:
-    #Expected if service unreachable
-    pass
-  finally:
-    s.close()
-    return is_service_socket_valid

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/files/alert_hive_thrift_port.py

@@ -23,26 +23,37 @@ import socket
 import time
 import traceback
 import urllib2
-from resource_management.libraries.functions import hive_check 
+from resource_management.libraries.functions import hive_check
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.core.environment import Environment
 
 OK_MESSAGE = "TCP OK - %.4f response on port %s"
 CRITICAL_MESSAGE = "Connection failed on host {0}:{1}"
 
 HIVE_SERVER_THRIFT_PORT_KEY = '{{hive-site/hive.server2.thrift.port}}'
 SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
+HIVE_SERVER2_AUTHENTICATION_KEY = '{{hive-site/hive.server2.authentication}}'
+HIVE_SERVER_PRINCIPAL_KEY = '{{hive-site/hive.server2.authentication.kerberos.principal}}'
+SMOKEUSER_KEYTAB_KEY = '{{cluster-env/smokeuser_keytab}}'
+SMOKEUSER_KEY = '{{cluster-env/smokeuser}}'
 
 PERCENT_WARNING = 200
 PERCENT_CRITICAL = 200
 
 THRIFT_PORT_DEFAULT = 10000
+HIVE_SERVER_PRINCIPAL_DEFAULT = 'hive/_HOST@EXAMPLE.COM'
+HIVE_SERVER2_AUTHENTICATION_DEFAULT = 'NOSASL'
+SMOKEUSER_KEYTAB_DEFAULT = '/etc/security/keytabs/smokeuser.headless.keytab'
+SMOKEUSER_DEFAULT = 'ambari-qa'
 
 def get_tokens():
   """
   Returns a tuple of tokens in the format {{site/property}} that will be used
   to build the dictionary passed into execute
   """
-  return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY)      
-  
+  return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY,SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY)
+
 
 def execute(parameters=None, host_name=None):
   """
@@ -58,22 +69,49 @@ def execute(parameters=None, host_name=None):
 
   thrift_port = THRIFT_PORT_DEFAULT
   if HIVE_SERVER_THRIFT_PORT_KEY in parameters:
-    thrift_port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY])  
+    thrift_port = int(parameters[HIVE_SERVER_THRIFT_PORT_KEY])
 
   security_enabled = False
   if SECURITY_ENABLED_KEY in parameters:
-    security_enabled = bool(parameters[SECURITY_ENABLED_KEY])  
+    security_enabled = str(parameters[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
+
+  hive_server2_authentication = HIVE_SERVER2_AUTHENTICATION_DEFAULT
+  if HIVE_SERVER2_AUTHENTICATION_KEY in parameters:
+    hive_server2_authentication = parameters[HIVE_SERVER2_AUTHENTICATION_KEY]
+
+  smokeuser = SMOKEUSER_DEFAULT
+  if SMOKEUSER_KEY in parameters:
+    smokeuser = parameters[SMOKEUSER_KEY]
 
   result_code = None
 
+  if security_enabled:
+    hive_server_principal = HIVE_SERVER_PRINCIPAL_DEFAULT
+    if HIVE_SERVER_PRINCIPAL_KEY in parameters:
+      hive_server_principal = parameters[HIVE_SERVER_PRINCIPAL_KEY]
+    smokeuser_keytab = SMOKEUSER_KEYTAB_DEFAULT
+    if SMOKEUSER_KEYTAB_KEY in parameters:
+      smokeuser_keytab = parameters[SMOKEUSER_KEYTAB_KEY]
+    with Environment() as env:
+      kinit_path_local = get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+      kinitcmd=format("{kinit_path_local} -kt {smokeuser_keytab} {smokeuser}; ")
+  else:
+    hive_server_principal = None
+    kinitcmd=None
+
   try:
     if host_name is None:
       host_name = socket.getfqdn()
 
     start_time = time.time()
-    is_thrift_port_ok = hive_check.check_thrift_port_sasl(host_name,
-        thrift_port, security_enabled=security_enabled)
-     
+    try:
+      with Environment() as env:
+        hive_check.check_thrift_port_sasl(host_name, thrift_port, hive_server2_authentication,
+                                          hive_server_principal, kinitcmd, smokeuser)
+      is_thrift_port_ok = True
+    except:
+      is_thrift_port_ok = False
+
     if is_thrift_port_ok == True:
       result_code = 'OK'
       total_time = time.time() - start_time
@@ -85,5 +123,5 @@ def execute(parameters=None, host_name=None):
   except Exception, e:
     label = str(e)
     result_code = 'UNKNOWN'
-        
+
   return ((result_code, [label]))

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py

@@ -68,12 +68,18 @@ def hive_service(
 
       is_service_socket_valid = False
       print "Waiting for the Hive server to start..."
+      if params.security_enabled:
+        kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
+      else:
+        kinitcmd=None
       while time.time() < end_time:
-        if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled):
+        try:
+          check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+                                 params.hive_server_principal, kinitcmd, params.smokeuser)
           is_service_socket_valid = True
           break
-        else:
-          time.sleep(2)
+        except:
+          time.sleep(5)
 
       elapsed_time = time.time() - start_time
       

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/params.py

@@ -56,6 +56,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 hive_server_host = config['clusterHostInfo']['hive_server_host'][0]
 hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
 hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}")
+hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
+hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication']
 
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 smoke_test_sql = format("{tmp_dir}/hiveserver2.sql")

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py

@@ -33,9 +33,16 @@ class HiveServiceCheck(Script):
     address=format("{hive_server_host}")
     port=int(format("{hive_server_port}"))
     print "Test connectivity to hive server"
-    if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled):
-      print "Successfully connected to %s on port %s" % (address, port)
+    if params.security_enabled:
+      kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
     else:
+      kinitcmd=None
+
+    try:
+      check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+                             params.hive_server_principal, kinitcmd, params.smokeuser)
+      print "Successfully connected to %s on port %s" % (address, port)
+    except:
       print "Connection to %s on port %s failed" % (address, port)
       exit(1)
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/files/alert_hive_thrift_port.py

@@ -23,25 +23,36 @@ import socket
 import time
 import traceback
 import urllib2
-from resource_management.libraries.functions import hive_check 
+from resource_management.libraries.functions import hive_check
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.core.environment import Environment
 
 OK_MESSAGE = "TCP OK - %.4f response on port %s"
 CRITICAL_MESSAGE = "Connection failed on host {0}:{1}"
 
 HIVE_SERVER_THRIFT_PORT_KEY = '{{hive-site/hive.server2.thrift.port}}'
 SECURITY_ENABLED_KEY = '{{cluster-env/security_enabled}}'
+HIVE_SERVER2_AUTHENTICATION_KEY = '{{hive-site/hive.server2.authentication}}'
+HIVE_SERVER_PRINCIPAL_KEY = '{{hive-site/hive.server2.authentication.kerberos.principal}}'
+SMOKEUSER_KEYTAB_KEY = '{{cluster-env/smokeuser_keytab}}'
+SMOKEUSER_KEY = '{{cluster-env/smokeuser}}'
 
 PERCENT_WARNING = 200
 PERCENT_CRITICAL = 200
 
 THRIFT_PORT_DEFAULT = 10000
+HIVE_SERVER_PRINCIPAL_DEFAULT = 'hive/_HOST@EXAMPLE.COM'
+HIVE_SERVER2_AUTHENTICATION_DEFAULT = 'NOSASL'
+SMOKEUSER_KEYTAB_DEFAULT = '/etc/security/keytabs/smokeuser.headless.keytab'
+SMOKEUSER_DEFAULT = 'ambari-qa'
 
 def get_tokens():
   """
   Returns a tuple of tokens in the format {{site/property}} that will be used
   to build the dictionary passed into execute
   """
-  return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY)      
+  return (HIVE_SERVER_THRIFT_PORT_KEY,SECURITY_ENABLED_KEY,HIVE_SERVER2_AUTHENTICATION_KEY,HIVE_SERVER_PRINCIPAL_KEY,SMOKEUSER_KEYTAB_KEY,SMOKEUSER_KEY)
   
 
 def execute(parameters=None, host_name=None):
@@ -62,18 +73,45 @@ def execute(parameters=None, host_name=None):
 
   security_enabled = False
   if SECURITY_ENABLED_KEY in parameters:
-    security_enabled = bool(parameters[SECURITY_ENABLED_KEY])  
+    security_enabled = str(parameters[SECURITY_ENABLED_KEY]).upper() == 'TRUE'
+
+  hive_server2_authentication = HIVE_SERVER2_AUTHENTICATION_DEFAULT
+  if HIVE_SERVER2_AUTHENTICATION_KEY in parameters:
+    hive_server2_authentication = parameters[HIVE_SERVER2_AUTHENTICATION_KEY]
+
+  smokeuser = SMOKEUSER_DEFAULT
+  if SMOKEUSER_KEY in parameters:
+    smokeuser = parameters[SMOKEUSER_KEY]
 
   result_code = None
 
+  if security_enabled:
+    hive_server_principal = HIVE_SERVER_PRINCIPAL_DEFAULT
+    if HIVE_SERVER_PRINCIPAL_KEY in parameters:
+      hive_server_principal = parameters[HIVE_SERVER_PRINCIPAL_KEY]
+    smokeuser_keytab = SMOKEUSER_KEYTAB_DEFAULT
+    if SMOKEUSER_KEYTAB_KEY in parameters:
+      smokeuser_keytab = parameters[SMOKEUSER_KEYTAB_KEY]
+    with Environment() as env:
+      kinit_path_local = get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+      kinitcmd=format("{kinit_path_local} -kt {smokeuser_keytab} {smokeuser}; ")
+  else:
+    hive_server_principal = None
+    kinitcmd=None
+
   try:
     if host_name is None:
       host_name = socket.getfqdn()
 
     start_time = time.time()
-    is_thrift_port_ok = hive_check.check_thrift_port_sasl(host_name,
-        thrift_port, security_enabled=security_enabled)
-     
+    try:
+      with Environment() as env:
+        hive_check.check_thrift_port_sasl(host_name, thrift_port, hive_server2_authentication,
+                                          hive_server_principal, kinitcmd, smokeuser)
+      is_thrift_port_ok = True
+    except:
+      is_thrift_port_ok = False
+
     if is_thrift_port_ok == True:
       result_code = 'OK'
       total_time = time.time() - start_time

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py

@@ -74,14 +74,20 @@ def hive_service(
 
       is_service_socket_valid = False
       print "Waiting for the Hive server to start..."
+      if params.security_enabled:
+        kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
+      else:
+        kinitcmd=None
       while time.time() < end_time:
-        if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled):
+        try:
+          check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+                                 params.hive_server_principal, kinitcmd, params.smokeuser)
           is_service_socket_valid = True
           break
-        else:
-          time.sleep(2)
+        except Exception, e:
+          time.sleep(5)
 
-      elapsed_time = time.time() - start_time    
+      elapsed_time = time.time() - start_time
       
       if is_service_socket_valid == False: 
         raise Fail("Connection to Hive server %s on port %s failed after %d seconds" % (address, port, elapsed_time))

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py

@@ -107,6 +107,8 @@ ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
 hive_server_host = config['clusterHostInfo']['hive_server_host'][0]
 hive_server_port = default('/configurations/hive-site/hive.server2.thrift.port',"10000")
 hive_url = format("jdbc:hive2://{hive_server_host}:{hive_server_port}")
+hive_server_principal = config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
+hive_server2_authentication = config['configurations']['hive-site']['hive.server2.authentication']
 
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 smoke_test_sql = format("{tmp_dir}/hiveserver2.sql")

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py

@@ -33,9 +33,16 @@ class HiveServiceCheck(Script):
     address=format("{hive_server_host}")
     port=int(format("{hive_server_port}"))
     print "Test connectivity to hive server"
-    if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled):
-      print "Successfully connected to %s on port %s" % (address, port)
+    if params.security_enabled:
+      kinitcmd=format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}; ")
     else:
+      kinitcmd=None
+
+    try:
+      check_thrift_port_sasl(address, port, params.hive_server2_authentication,
+                             params.hive_server_principal, kinitcmd, params.smokeuser)
+      print "Successfully connected to %s on port %s" % (address, port)
+    except:
       print "Connection to %s on port %s failed" % (address, port)
       exit(1)
 

ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_server.py

@@ -50,7 +50,7 @@ class TestHiveServer(RMFTestCase):
   @patch("socket.socket")
   def test_start_default(self, socket_mock):
     s = socket_mock.return_value
-    
+
     self.executeScript("1.3.2/services/HIVE/package/scripts/hive_server.py",
                          classname = "HiveServer",
                          command = "start",
@@ -123,10 +123,8 @@ class TestHiveServer(RMFTestCase):
     )
     self.assertNoMoreResources()
 
-  @patch("socket.socket")
-  def test_start_secured(self, socket_mock):
-    s = socket_mock.return_value
-    
+  def test_start_secured(self):
+
     self.executeScript("1.3.2/services/HIVE/package/scripts/hive_server.py",
                        classname = "HiveServer",
                        command = "start",
@@ -156,10 +154,15 @@ class TestHiveServer(RMFTestCase):
     self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
     )
-
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',
+                              user = 'ambari-qa',
+    )
+    self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'",
+                              path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+                              user = 'ambari-qa',
+                              timeout = 30,
+                              )
     self.assertNoMoreResources()
-    self.assertTrue(socket_mock.called)    
-    self.assertTrue(s.close.called)
 
   @patch("socket.socket")
   def test_stop_secured(self, socket_mock):

ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py

@@ -80,6 +80,14 @@ class TestServiceCheck(RMFTestCase):
                         command="service_check",
                         config_file="secured.json"
     )
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',
+                              user = 'ambari-qa',
+                              )
+    self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'",
+                              path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+                              user = 'ambari-qa',
+                              timeout = 30,
+                              )
     self.assertResourceCalled('File', '/tmp/hcatSmoke.sh',
                         content = StaticFile('hcatSmoke.sh'),
                         mode = 0755,
@@ -117,5 +125,4 @@ class TestServiceCheck(RMFTestCase):
                               tries = 3,
                               try_sleep = 5,
                               )
-    self.assertNoMoreResources()
-    self.assertTrue(socket_mock.called)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/1.3.2/configs/default.json

@@ -281,7 +281,8 @@
             "ambari.hive.db.schema.name": "hive", 
             "hive.metastore.execute.setugi": "true", 
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
-            "hive.server2.enable.doAs": "true", 
+            "hive.server2.enable.doAs": "true",
+            "hive.server2.authentication": "NOSASL",
             "hive.optimize.mapjoin.mapreduce": "true"
         },
         "webhcat-env": {

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_server.py

@@ -21,6 +21,7 @@ import os
 import subprocess
 from mock.mock import MagicMock, call, patch
 from resource_management.core import shell
+from resource_management.libraries.functions import hive_check
 from stacks.utils.RMFTestCase import *
 
 import socket
@@ -115,11 +116,16 @@ class TestHiveServer(RMFTestCase):
     self.assertResourceCalled('Execute', '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/share/java/mysql-connector-java.jar org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\' hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
                               path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'], tries=5, try_sleep=10
     )
-
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',
+                              user = 'ambari-qa',
+                              )
+    self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'",
+                              path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+                              user = 'ambari-qa',
+                              timeout = 30,
+                              )
     self.assertNoMoreResources()
     self.assertTrue(check_fs_root_mock.called)
-    self.assertTrue(socket_mock.called)
-    self.assertTrue(s.close.called)
 
   @patch("socket.socket")
   def test_stop_secured(self, socket_mock):

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py

@@ -82,6 +82,14 @@ class TestServiceCheck(RMFTestCase):
                         command="service_check",
                         config_file="secured.json"
     )
+    self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa; ',
+                              user = 'ambari-qa',
+                              )
+    self.assertResourceCalled('Execute', "! beeline -u 'jdbc:hive2://c6402.ambari.apache.org:10000/;principal=hive/_HOST@EXAMPLE.COM' -e '' 2>&1| awk '{print}'|grep -i -e 'Connection refused' -e 'Invalid URL'",
+                              path = ['/bin/', '/usr/bin/', '/usr/lib/hive/bin/', '/usr/sbin/'],
+                              user = 'ambari-qa',
+                              timeout = 30,
+                              )
     self.assertResourceCalled('File', '/tmp/hcatSmoke.sh',
                         content = StaticFile('hcatSmoke.sh'),
                         mode = 0755,
@@ -121,5 +129,4 @@ class TestServiceCheck(RMFTestCase):
                               tries = 3,
                               try_sleep = 5,
                               )
-    self.assertNoMoreResources()
-    self.assertTrue(socket_mock.called)
+    self.assertNoMoreResources()

ambari-server/src/test/python/stacks/2.0.6/configs/default.json

@@ -368,7 +368,8 @@
             "ambari.hive.db.schema.name": "hive", 
             "hive.metastore.execute.setugi": "true", 
             "hive.auto.convert.sortmerge.join.noconditionaltask": "true", 
-            "hive.server2.enable.doAs": "true", 
+            "hive.server2.enable.doAs": "true",
+            "hive.server2.authentication": "NOSASL",
             "hive.optimize.mapjoin.mapreduce": "true"
         }, 
         "yarn-site": {