AMBARI-7733. Alerts on Hiveserver2 (aonishuk)

ambari-common/src/main/python/resource_management/libraries/functions/hive_check.py

@@ -20,7 +20,7 @@ limitations under the License.
 import socket
 from resource_management.core.exceptions import Fail
 
-def check_thrift_port_sasl(address, port, timeout = 5):
+def check_thrift_port_sasl(address, port, timeout = 5, security_enabled = False):
   """
   Hive thrift SASL port check
   """
@@ -73,8 +73,13 @@ def check_thrift_port_sasl(address, port, timeout = 5):
 
   try:
     s.connect((address, port))
-    s.send(msg)
+    #Successfull connection, port check passed
     is_service_socket_valid = True
+
+    # Try to send anonymous plain auth message to thrift to prevent errors in hive log
+    # Plain mechanism is not supported in security mode
+    if not security_enabled:
+      s.send(msg)
   except socket.error, e:
     #Expected if service unreachable
     pass

ambari-common/src/main/unix/ambari-python-wrap

@@ -17,7 +17,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-export PYTHONPATH=/usr/lib/python2.6/site-packages/ambari_commons:$PYTHONPATH
+export PYTHONPATH=/usr/lib/python2.6/site-packages:/usr/lib/python2.6/site-packages/ambari_commons:$PYTHONPATH
 
 # reset settings
 unset PYTHON

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive_service.py

@@ -69,7 +69,7 @@ def hive_service(
       is_service_socket_valid = False
       print "Waiting for the Hive server to start..."
       while time.time() < end_time:
-        if check_thrift_port_sasl(address, port, 2):
+        if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled):
           is_service_socket_valid = True
           break
         else:

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/service_check.py

@@ -33,10 +33,11 @@ class HiveServiceCheck(Script):
     address=format("{hive_server_host}")
     port=int(format("{hive_server_port}"))
     print "Test connectivity to hive server"
-    if check_thrift_port_sasl(address, port):
+    if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled):
       print "Successfully connected to %s on port %s" % (address, port)
     else:
-      print "Connection to %s on port %s failed: %s" % (address, port)
+      print "Connection to %s on port %s failed" % (address, port)
+      exit(1)
 
     hcat_service_check()
     webhcat_service_check()

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/files/check_hive_thrift_port.py

@@ -37,22 +37,28 @@ def main():
 
   parser.add_option("-H", "--host", dest="address", help="Hive thrift host")
   parser.add_option("-p", "--port", type="int", dest="port", help="Hive thrift port")
+  parser.add_option("--security-enabled", action="store_true", dest="security_enabled")
 
   (options, args) = parser.parse_args()
 
   if options.address is None:
-    print "Hive thrift host (--name or -n)"
+    print "Specify hive thrift host (--host or -H)"
     exit(-1)
 
   if options.port is None:
-    print "Hive thrift port (--file or -f)"
+    print "Specify hive thrift port (--port or -p)"
     exit(-1)
 
+  if options.security_enabled:
+    security_enabled = options.security_enabled
+  else:
+    security_enabled = False
+
   address = options.address
   port = options.port
 
   starttime = time()
-  if check_thrift_port_sasl(address, port):
+  if check_thrift_port_sasl(address, port, security_enabled=security_enabled):
     timetaken = time() - starttime
     print OK_MESSAGE % (timetaken, port)
     exit(0)

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-commands.cfg.j2

@@ -143,5 +143,5 @@ define command{
 
 define command{
         command_name check_tcp_wrapper_sasl
-        command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$
+        command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$ $ARG2$
        }

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/NAGIOS/package/templates/hadoop-services.cfg.j2

@@ -539,7 +539,7 @@ define service {
         use                     hadoop-service
         service_description     HIVE-SERVER::HiveServer2 process
         servicegroups           HIVE
-        check_command           check_tcp_wrapper_sasl!{{ hive_server_port }}!-w 1 -c 1
+        check_command           check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ '--security-enabled' if security_enabled else '' }}!-w 1 -c 1
         normal_check_interval   0.5
         retry_check_interval    0.5
         max_check_attempts      3

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_service.py

@@ -75,7 +75,7 @@ def hive_service(
       is_service_socket_valid = False
       print "Waiting for the Hive server to start..."
       while time.time() < end_time:
-        if check_thrift_port_sasl(address, port, 2):
+        if check_thrift_port_sasl(address, port, 2, security_enabled=params.security_enabled):
           is_service_socket_valid = True
           break
         else:

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/service_check.py

@@ -33,10 +33,11 @@ class HiveServiceCheck(Script):
     address=format("{hive_server_host}")
     port=int(format("{hive_server_port}"))
     print "Test connectivity to hive server"
-    if check_thrift_port_sasl(address, port):
+    if check_thrift_port_sasl(address, port, security_enabled=params.security_enabled):
       print "Successfully connected to %s on port %s" % (address, port)
     else:
-      print "Connection to %s on port %s failed: %s" % (address, port)
+      print "Connection to %s on port %s failed" % (address, port)
+      exit(1)
 
     hcat_service_check()
     webhcat_service_check()

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/files/check_hive_thrift_port.py

@@ -37,22 +37,28 @@ def main():
 
   parser.add_option("-H", "--host", dest="address", help="Hive thrift host")
   parser.add_option("-p", "--port", type="int", dest="port", help="Hive thrift port")
+  parser.add_option("--security-enabled", action="store_true", dest="security_enabled")
 
   (options, args) = parser.parse_args()
 
   if options.address is None:
-    print "Hive thrift host (--name or -n)"
+    print "Specify hive thrift host (--host or -H)"
     exit(-1)
 
   if options.port is None:
-    print "Hive thrift port (--file or -f)"
+    print "Specify hive thrift port (--port or -p)"
     exit(-1)
 
+  if options.security_enabled:
+    security_enabled = options.security_enabled
+  else:
+    security_enabled = False
+
   address = options.address
   port = options.port
 
   starttime = time()
-  if check_thrift_port_sasl(address, port):
+  if check_thrift_port_sasl(address, port, security_enabled=security_enabled):
     timetaken = time() - starttime
     print OK_MESSAGE % (timetaken, port)
     exit(0)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-commands.cfg.j2

@@ -157,7 +157,7 @@ define command{
 
 define command{
         command_name check_tcp_wrapper_sasl
-        command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$
+        command_line $USER1$/check_wrapper.sh /var/lib/ambari-agent/ambari-python-wrap $USER1$/check_hive_thrift_port.py -H $HOSTADDRESS$ -p $ARG1$ $ARG2$
        }
 
 define command{

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/NAGIOS/package/templates/hadoop-services.cfg.j2

@@ -700,7 +700,7 @@ define service {
         use                     hadoop-service
         service_description     HIVE-SERVER::HiveServer2 process
         servicegroups           HIVE
-        check_command           check_tcp_wrapper_sasl!{{ hive_server_port }}!-w 1 -c 1
+        check_command           check_tcp_wrapper_sasl!{{ hive_server_port }}!{{ '--security-enabled' if security_enabled else '' }}!-w 1 -c 1
         normal_check_interval   0.5
         retry_check_interval    0.5
         max_check_attempts      3

ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hive_service_check.py

@@ -22,11 +22,11 @@ from stacks.utils.RMFTestCase import *
 import datetime, socket
 import  resource_management.libraries.functions
 @patch.object(resource_management.libraries.functions, "get_unique_id_and_date", new = MagicMock(return_value=''))
-@patch("socket.socket", new = MagicMock())
+@patch("socket.socket")
 class TestServiceCheck(RMFTestCase):
 
   @patch("sys.exit")
-  def test_service_check_default(self, sys_exit_mock):
+  def test_service_check_default(self, sys_exit_mock, socket_mock):
 
     self.executeScript("1.3.2/services/HIVE/package/scripts/service_check.py",
                         classname="HiveServiceCheck",
@@ -70,9 +70,10 @@ class TestServiceCheck(RMFTestCase):
                               try_sleep = 5,
                               )
     self.assertNoMoreResources()
+    self.assertTrue(socket_mock.called)
 
   @patch("sys.exit")
-  def test_service_check_secured(self, sys_exit_mock):
+  def test_service_check_secured(self, sys_exit_mock, socket_mock):
 
     self.executeScript("1.3.2/services/HIVE/package/scripts/service_check.py",
                         classname="HiveServiceCheck",
@@ -117,3 +118,4 @@ class TestServiceCheck(RMFTestCase):
                               try_sleep = 5,
                               )
     self.assertNoMoreResources()
+    self.assertTrue(socket_mock.called)

ambari-server/src/test/python/stacks/2.0.6/HIVE/test_hive_service_check.py

@@ -23,11 +23,11 @@ from stacks.utils.RMFTestCase import *
 import datetime, sys, socket
 import  resource_management.libraries.functions
 @patch.object(resource_management.libraries.functions, "get_unique_id_and_date", new = MagicMock(return_value=''))
-@patch("socket.socket", new = MagicMock())
+@patch("socket.socket")
 class TestServiceCheck(RMFTestCase):
 
   @patch("sys.exit")
-  def test_service_check_default(self, sys_exit_mock):
+  def test_service_check_default(self, sys_exit_mock, socket_mock):
 
     self.executeScript("2.0.6/services/HIVE/package/scripts/service_check.py",
                         classname="HiveServiceCheck",
@@ -72,9 +72,10 @@ class TestServiceCheck(RMFTestCase):
                               try_sleep = 5,
                               )
     self.assertNoMoreResources()
+    self.assertTrue(socket_mock.called)
 
   @patch("sys.exit")
-  def test_service_check_secured(self, sys_exit_mock):
+  def test_service_check_secured(self, sys_exit_mock, socket_mock):
 
     self.executeScript("2.0.6/services/HIVE/package/scripts/service_check.py",
                         classname="HiveServiceCheck",
@@ -121,4 +122,4 @@ class TestServiceCheck(RMFTestCase):
                               try_sleep = 5,
                               )
     self.assertNoMoreResources()
-    self.assertNoMoreResources()
+    self.assertTrue(socket_mock.called)