AMBARI-12998. Add HiveServer2, Metastore, and Client Heap Sizes to Smart Configs (alexantonenko)

ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml

@@ -25,12 +25,28 @@
     <name>hive.client.heapsize</name>
     <value>512</value>
     <description>Hive Client Java heap size</description>
+    <display-name>Client Heap Size</display-name>
+    <value-attributes>
+      <type>int</type>
+      <minimum>512</minimum>
+      <maximum>2048</maximum>
+      <unit>MB</unit>
+      <increment-step>512</increment-step>
+    </value-attributes>
   </property>
 
   <property>
     <name>hive.metastore.heapsize</name>
     <value>1024</value>
     <description>Hive Metastore Java heap size</description>
+    <display-name>Metastore Heap Size</display-name>
+    <value-attributes>
+      <type>int</type>
+      <minimum>512</minimum>
+      <maximum>2048</maximum>
+      <unit>MB</unit>
+      <increment-step>512</increment-step>
+    </value-attributes>
   </property>
 
   <property>

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/hive-env.xml

@@ -23,8 +23,16 @@
 <configuration>
   <property>
     <name>hive.heapsize</name>
-    <value>1024</value>
+    <value>512</value>
     <description>Hive Java heap size</description>
+    <display-name>HiveServer2 Heap Size</display-name>
+    <value-attributes>
+      <type>int</type>
+      <minimum>512</minimum>
+      <maximum>2048</maximum>
+      <unit>MB</unit>
+      <increment-step>512</increment-step>
+    </value-attributes>
   </property>
 
   <!-- hive-env.sh -->

ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/themes/theme.json

@@ -149,6 +149,18 @@
         },
         {
           "config": "hive-env/hive_exec_orc_storage_strategy",
+          "subsection-name": "optimization-row2-col1"
+        },
+        {
+          "config": "hive-env/hive.heapsize",
+          "subsection-name": "optimization-row2-col2"
+        },
+        {
+          "config": "hive-env/hive.metastore.heapsize",
+          "subsection-name": "optimization-row2-col2"
+        },
+        {
+          "config": "hive-env/hive.client.heapsize",
           "subsection-name": "optimization-row2-col2"
         },
         {
@@ -249,6 +261,39 @@
           "type": "toggle"
         }
       },
+      {
+        "config": "hive-env/hive.heapsize",
+        "widget": {
+          "type": "slider",
+          "units": [
+            {
+              "unit-name": "MB"
+            }
+          ]
+        }
+      },
+      {
+        "config": "hive-env/hive.metastore.heapsize",
+        "widget": {
+          "type": "slider",
+          "units": [
+            {
+              "unit-name": "MB"
+            }
+          ]
+        }
+      },
+      {
+        "config": "hive-env/hive.client.heapsize",
+        "widget": {
+          "type": "slider",
+          "units": [
+            {
+              "unit-name": "MB"
+            }
+          ]
+        }
+      },
       {
         "config": "hive-site/hive.auto.convert.join.noconditionaltask.size",
         "widget": {

ambari-server/src/main/resources/stacks/HDP/2.2/services/stack_advisor.py

@@ -242,7 +242,7 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
     putHiveEnvProperty = self.putProperty(configurations, "hive-env", services)
     putHiveSiteProperty = self.putProperty(configurations, "hive-site", services)
     putHiveSitePropertyAttribute = self.putPropertyAttribute(configurations, "hive-site")
-
+    putHiveEnvPropertyAttributes = self.putPropertyAttribute(configurations, "hive-env")
     servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
 
     #  Storage
@@ -475,6 +475,24 @@ class HDP22StackAdvisor(HDP21StackAdvisor):
               ("hive-site" in services["configurations"] and "hive.server2.custom.authentication.class" in services["configurations"]["hive-site"]["properties"]):
         putHiveSitePropertyAttribute("hive.server2.custom.authentication.class", "delete", "true")
 
+    # HiveServer, Client, Metastore heapsize
+    hs_heapsize_multiplier = 3.0/8
+    hm_heapsize_multiplier = 1.0/8
+    # HiveServer2 and HiveMetastore located on the same host
+    hive_server_hosts = self.getHostsWithComponent("HIVE", "HIVE_SERVER", services, hosts)
+    hive_client_hosts = self.getHostsWithComponent("HIVE", "HIVE_CLIENT", services, hosts)
+
+    if hive_server_hosts is not None and len(hive_server_hosts):
+      hs_host_ram = hive_server_hosts[0]["Hosts"]["total_mem"]/1024
+      putHiveEnvProperty("hive.metastore.heapsize", max(512, int(hs_host_ram*hm_heapsize_multiplier)))
+      putHiveEnvProperty("hive.heapsize", max(512, int(hs_host_ram*hs_heapsize_multiplier)))
+      putHiveEnvPropertyAttributes("hive.metastore.heapsize", "maximum", max(1024, hs_host_ram))
+      putHiveEnvPropertyAttributes("hive.heapsize", "maximum", max(1024, hs_host_ram))
+
+    if hive_client_hosts is not None and len(hive_client_hosts):
+      putHiveEnvProperty("hive.client.heapsize", 1024)
+      putHiveEnvPropertyAttributes("hive.client.heapsize", "maximum", max(1024, int(hive_client_hosts[0]["Hosts"]["total_mem"]/1024)))
+
 
   def recommendHBASEConfigurations(self, configurations, clusterData, services, hosts):
     super(HDP22StackAdvisor, self).recommendHbaseConfigurations(configurations, clusterData, services, hosts)

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1149,6 +1149,80 @@ class TestHDP22StackAdvisor(TestCase):
       "changed-configurations": [ ]
     }
 
+    hiveService = {
+      "services": [
+        {
+          "href": "/api/v1/stacks/HDP/versions/2.2/services/HIVE",
+          "StackServices": {
+            "service_name": "HIVE",
+            "service_version": "2.6.0.2.2",
+            "stack_name": "HDP",
+            "stack_version": "2.2"
+          },
+          "components": [
+            {
+              "StackServiceComponents": {
+                "advertise_version": "false",
+                "cardinality": "1",
+                "component_category": "MASTER",
+                "component_name": "HIVE_SERVER",
+                "display_name": "HiveServer2",
+                "is_client": "false",
+                "is_master": "true",
+                "hostnames": [
+                  "c6402.ambari.apache.org"
+                ]
+              },
+              "dependencies": []
+            },
+            {
+              "StackServiceComponents": {
+                "advertise_version": "true",
+                "cardinality": "1+",
+                "component_category": "SLAVE",
+                "component_name": "HIVE_CLIENT",
+                "display_name": "Hive Client",
+                "is_client": "true",
+                "is_master": "false",
+                "hostnames": [
+                  "c6402.ambari.apache.org",
+                  "c6403.ambari.apache.org"
+                ]
+              },
+              "dependencies": []
+            }
+          ]
+        },
+      ],
+      "configurations": {
+        "hive-env": {
+          "properties": {
+            "hive.heapsize": "200",
+            "hive.metastore.heapsize": "200",
+            "hive.client.heapsize": "200"
+          }
+        },
+        "hive-site": {
+          "properties": {
+            "hive.server2.authentication": "none",
+            "hive.server2.authentication.ldap.url": "",
+            "hive.server2.authentication.ldap.baseDN": "",
+            "hive.server2.authentication.kerberos.keytab": "",
+            "hive.server2.authentication.kerberos.principal": "",
+            "hive.server2.authentication.pam.services": "",
+            "hive.server2.custom.authentication.class": ""
+          }
+        },
+        "hiveserver2-site": {
+          "properties": {
+            "hive.security.authorization.manager": "",
+            "hive.security.authenticator.manager": ""
+          }
+        }
+      },
+      "changed-configurations": [ ]
+    }
+
     hosts = {
       "items" : [
         {
@@ -1286,6 +1360,19 @@ class TestHDP22StackAdvisor(TestCase):
     self.assertEquals(configurations['hive-site']['property_attributes']['hive.server2.tez.default.queues'], expected['hive-site']['property_attributes']['hive.server2.tez.default.queues'])
     self.assertEquals(configurations['hive-site']['properties']['hive.server2.tez.default.queues'], expected['hive-site']['properties']['hive.server2.tez.default.queues'])
 
+    # Hive heapsize properties
+    self.stackAdvisor.recommendHIVEConfigurations(configurations, clusterData, hiveService, hosts)
+
+    # Recommended default values
+    self.assertEquals(configurations["hive-env"]["properties"]["hive.metastore.heapsize"], "512")
+    self.assertEquals(configurations["hive-env"]["properties"]["hive.heapsize"], "703")
+    self.assertEquals(configurations["hive-env"]["properties"]["hive.client.heapsize"], "1024")
+
+    # Recommended attributes for maximum values, minimum values defined in stack definition
+    self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.heapsize"]["maximum"], "1877")
+    self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.metastore.heapsize"]["maximum"], "1877")
+    self.assertEquals(configurations["hive-env"]["property_attributes"]["hive.client.heapsize"]["maximum"], "1877")
+
   def test_recommendMapredConfigurationAttributesWithPigService(self):
     configurations = {
       "mapred-site": {