AMBARI-18157. Hive service check is failing after cluster Kerberization (dgrinenko via dlysnichenko)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -1676,6 +1676,97 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
             leafQueueNames.add(leafQueueName)
     return leafQueueNames
 
+  def get_service_component_meta(self, service, component, services):
+    """
+    Function retrieve service component meta information as dict from services.json
+    If no service or component found, would be returned empty dict
+
+    Return value example:
+        "advertise_version" : true,
+        "bulk_commands_display_name" : "",
+        "bulk_commands_master_component_name" : "",
+        "cardinality" : "1+",
+        "component_category" : "CLIENT",
+        "component_name" : "HBASE_CLIENT",
+        "custom_commands" : [ ],
+        "decommission_allowed" : false,
+        "display_name" : "HBase Client",
+        "has_bulk_commands_definition" : false,
+        "is_client" : true,
+        "is_master" : false,
+        "reassign_allowed" : false,
+        "recovery_enabled" : false,
+        "service_name" : "HBASE",
+        "stack_name" : "HDP",
+        "stack_version" : "2.5",
+        "hostnames" : [ "host1", "host2" ]
+
+    :type service str
+    :type component str
+    :type services dict
+    :rtype dict
+    """
+    __stack_services = "StackServices"
+    __stack_service_components = "StackServiceComponents"
+
+    if not services:
+      return {}
+
+    service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+    if len(service_meta) == 0:
+      return {}
+
+    service_meta = service_meta[0]
+    component_meta = [item for item in service_meta["components"] if item[__stack_service_components]["component_name"] == component]
+
+    if len(component_meta) == 0:
+      return {}
+
+    return component_meta[0][__stack_service_components]
+
+  def is_secured_cluster(self, services):
+    """
+    Detects if cluster is secured or not
+    :type services dict
+    :rtype bool
+    """
+    return services and "cluster-env" in services["configurations"] and\
+           "security_enabled" in services["configurations"]["cluster-env"]["properties"] and\
+           services["configurations"]["cluster-env"]["properties"]["security_enabled"].lower() == "true"
+
+  def get_services_list(self, services):
+    """
+    Returns available services as list
+
+    :type services dict
+    :rtype list
+    """
+    if not services:
+      return []
+
+    return [service["StackServices"]["service_name"] for service in services["services"]]
+
+  def get_components_list(self, service, services):
+    """
+    Return list of components for specific service
+    :type service str
+    :type services dict
+    :rtype list
+    """
+    __stack_services = "StackServices"
+    __stack_service_components = "StackServiceComponents"
+
+    if not services:
+      return []
+
+    service_meta = [item for item in services["services"] if item[__stack_services]["service_name"] == service]
+    if len(service_meta) == 0:
+      return []
+
+    service_meta = service_meta[0]
+    return [item[__stack_service_components]["component_name"] for item in service_meta["components"]]
+
+
 def getOldValue(self, services, configType, propertyName):
   if services:
     if 'changed-configurations' in services.keys():

ambari-server/src/main/resources/stacks/HDP/2.1/services/stack_advisor.py

@@ -107,7 +107,7 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
           dbConnection = self.getDBConnectionString(hiveEnvProperties['hive_database']).format(hiveServerHost['Hosts']['host_name'], hiveSiteProperties['ambari.hive.db.schema.name'])
           putHiveProperty('javax.jdo.option.ConnectionURL', dbConnection)
 
-    servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+    servicesList = self.get_services_list(services)
     if "PIG" in servicesList:
         ambari_user = self.getAmbariUser(services)
         ambariHostName = socket.getfqdn()
@@ -120,6 +120,28 @@ class HDP21StackAdvisor(HDP206StackAdvisor):
             webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.hosts".format(old_ambari_user), 'delete', 'true')
             webHcatSitePropertyAttributes("webhcat.proxyuser.{0}.groups".format(old_ambari_user), 'delete', 'true')
 
+    if self.is_secured_cluster(services):
+      appendCoreSiteProperty = self.updateProperty(configurations, "core-site", services)
+
+      def updateCallback(originalValue, newValue):
+        """
+        :type originalValue str
+        :type newValue list
+        """
+        if originalValue and not originalValue.isspace():
+          hosts = originalValue.split(',')
+
+          if newValue:
+            hosts.extend(newValue)
+
+          result = ','.join(set(hosts))
+          return result
+        else:
+          return ','.join(set(newValue))
+
+      meta = self.get_service_component_meta("HIVE", "WEBHCAT_SERVER", services)
+      if "hostnames" in meta:
+        appendCoreSiteProperty('hadoop.proxyuser.HTTP.hosts', meta["hostnames"], updateCallback)
 
   def recommendTezConfigurations(self, configurations, clusterData, services, hosts):
     putTezProperty = self.putProperty(configurations, "tez-site")

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -188,6 +188,59 @@ class TestHDP206StackAdvisor(TestCase):
     ]
     self.assertValidationResult(expectedItems, result)
 
+  def test_get_components_list(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [
+          {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+          {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+        ]
+      }
+    ]
+    expected = sorted(["GANGLIA_MONITOR", "GANGLIA_SERVER"])
+
+    services = self.prepareServices(servicesInfo)
+    result = sorted(self.stackAdvisor.get_components_list("GANGLIA", services))
+
+    self.assertEqual(expected, result)
+
+  def test_get_services_list(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [
+          {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+          {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+        ]
+      }
+    ]
+    expected = ["GANGLIA"]
+
+    services = self.prepareServices(servicesInfo)
+    result = self.stackAdvisor.get_services_list(services)
+
+    self.assertEqual(expected, result)
+
+  def test_get_service_component_meta(self):
+    servicesInfo = [
+      {
+        "name": "GANGLIA",
+        "components": [
+          {"name": "GANGLIA_MONITOR", "display_name": "Ganglia Monitor", "cardinality": "1+", "category": "SLAVE", "is_master": False, "hostnames": ["host1"]},
+          {"name": "GANGLIA_SERVER", "display_name": "Ganglia Server", "cardinality": "3+", "category": "MASTER", "is_master": True, "hostnames": ["host2", "host1"]}
+        ]
+      }
+    ]
+    expected = ["host1"]
+
+    services = self.prepareServices(servicesInfo)
+    result = self.stackAdvisor.get_service_component_meta("GANGLIA", "GANGLIA_MONITOR", services)
+
+    self.assertEquals(True, "hostnames" in result)
+    self.assertEqual(expected, result["hostnames"])
+
+
   def test_validationWarnMessagesIfLessThanDefault(self):
     servicesInfo = [
       {

ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py

@@ -270,6 +270,89 @@ class TestHDP21StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-site']['properties']['javax.jdo.option.ConnectionURL'], "jdbc:mysql://example.com/hive_name")
     self.assertEquals(configurations['hive-site']['properties']['javax.jdo.option.ConnectionDriverName'], "com.mysql.jdbc.Driver")
 
+  def test_recommendHiveConfigurationsSecure(self):
+    services = {
+      "services": [
+        {
+          "StackServices": {
+            "service_name": "HIVE",
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "component_name": "WEBHCAT_SERVER",
+                "service_name": "HIVE",
+                "hostnames": ["example.com"]
+              }
+            }
+          ]
+        }
+      ],
+      "configurations": {}
+    }
+
+    configurations = {
+      "hive-site": {
+        "properties": {
+          "javax.jdo.option.ConnectionDriverName": "",
+          "ambari.hive.db.schema.name": "hive_name",
+          "javax.jdo.option.ConnectionURL": "jdbc:mysql://localhost/hive?createDatabaseIfNotExist=true"
+        }
+      },
+      "hive-env": {
+        "properties": {
+          "hive_database": "New MySQL Database"
+        }
+      },
+      "cluster-env": {
+        "properties": {
+          "security_enabled": "true"
+        }
+      }
+    }
+
+    services['configurations'] = configurations
+    clusterData = {
+      "mapMemory": 3000,
+      "reduceMemory": 2056,
+      "containers": 3,
+      "ramPerContainer": 256
+    }
+    services['changed-configurations'] = []
+    hosts = {
+      "items": [
+        {
+          "Hosts": {
+            "host_name": "example.com"
+          }
+        },
+        {
+          "Hosts": {
+            "host_name": "example.org"
+          }
+        }
+      ]
+    }
+
+    # new mysql
+    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals("core-site" in configurations, True)
+    self.assertEqual("hadoop.proxyuser.HTTP.hosts" in configurations["core-site"]["properties"], True)
+    self.assertEqual(configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] == "example.com", True)
+
+    newhost_list = ["example.com", "example.org"]
+    services["services"][0]["components"][0]["StackServiceComponents"]["hostnames"] = newhost_list
+    configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"] = ""
+
+    self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, services, hosts)
+    self.assertEquals("core-site" in configurations, True)
+    self.assertEqual("hadoop.proxyuser.HTTP.hosts" in configurations["core-site"]["properties"], True)
+
+    fetch_list = sorted(configurations["core-site"]["properties"]["hadoop.proxyuser.HTTP.hosts"].split(","))
+    self.assertEqual(sorted(newhost_list), fetch_list)
+
+
+
   def test_recommendHiveConfigurations_containersRamIsLess(self):
     configurations = {}
     clusterData = {