AMBARI-8989. Configs: HBase heap sizes should not be set to 1GB (dlysnichenko)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -83,7 +83,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     return {
       "YARN": self.recommendYARNConfigurations,
       "MAPREDUCE2": self.recommendMapReduce2Configurations,
-      "HDFS": self.recommendHDFSConfigurations
+      "HDFS": self.recommendHDFSConfigurations,
+      "HBASE": self.recommendHbaseEnvConfigurations
     }
 
   def putProperty(self, config, configType):
@@ -119,6 +120,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     putHDFSProperty = self.putProperty(configurations, "hadoop-env")
     putHDFSProperty('namenode_opt_maxnewsize', max(int(clusterData['totalAvailableRam'] / 8), 256))
 
+  def recommendHbaseEnvConfigurations(self, configurations, clusterData, services, hosts):
+    putHbaseProperty = self.putProperty(configurations, "hbase-env")
+    putHbaseProperty('hbase_regionserver_heapsize', int(clusterData['hbaseRam']) * 1024)
+    putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024)
+
   def getConfigurationClusterSummary(self, servicesList, hosts, components):
 
     hBaseInstalled = False
@@ -221,7 +227,8 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
     return {
       "HDFS": {"hadoop-env": self.validateHDFSConfigurationsEnv},
       "MAPREDUCE2": {"mapred-site": self.validateMapReduce2Configurations},
-      "YARN": {"yarn-site": self.validateYARNConfigurations}
+      "YARN": {"yarn-site": self.validateYARNConfigurations},
+      "HBASE": {"hbase-env": self.validateHbaseEnvConfigurations}
     }
 
   def validateServiceConfigurations(self, serviceName):
@@ -302,6 +309,11 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
                         {"config-name": 'yarn.scheduler.minimum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.minimum-allocation-mb')},
                         {"config-name": 'yarn.scheduler.maximum-allocation-mb', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'yarn.scheduler.maximum-allocation-mb')} ]
     return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+ 
+  def validateHbaseEnvConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
+    validationItems = [ {"config-name": 'hbase_regionserver_heapsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hbase_regionserver_heapsize')},
+                        {"config-name": 'hbase_master_heapsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'hbase_master_heapsize')} ]
+    return self.toConfigurationValidationProblems(validationItems, "hbase-env")
 
   def validateHDFSConfigurationsEnv(self, properties, recommendedDefaults, configurations, services, hosts):
     validationItems = [ {"config-name": 'namenode_heapsize', "item": self.validatorLessThenDefaultValue(properties, recommendedDefaults, 'namenode_heapsize')},

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -499,6 +499,43 @@ class TestHDP206StackAdvisor(TestCase):
       actualItems.append(next)
     self.checkEqual(expectedItems, actualItems)
 
+  def test_recommendHbaseEnvConfigurations(self):
+    servicesList = ["HBASE"]
+    configurations = {}
+    components = []
+    hosts = {
+      "items" : [
+        {
+          "Hosts" : {
+            "cpu_count" : 6,
+            "total_mem" : 50331648,
+            "disk_info" : [
+              {"mountpoint" : "/"},
+              {"mountpoint" : "/dev/shm"},
+              {"mountpoint" : "/vagrant"},
+              {"mountpoint" : "/"},
+              {"mountpoint" : "/dev/shm"},
+              {"mountpoint" : "/vagrant"}
+            ]
+          }
+        }
+      ]
+    }
+    expected = {
+      "hbase-env": {
+        "properties": {
+          "hbase_master_heapsize": "8192",
+          "hbase_regionserver_heapsize": "8192",
+          }
+      }
+    }
+
+    clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components)
+    self.assertEquals(clusterData['hbaseRam'], 8)
+
+    self.stackAdvisor.recommendHbaseEnvConfigurations(configurations, clusterData, None, None)
+    self.assertEquals(configurations, expected)
+
   def test_recommendHDFSConfigurations(self):
     configurations = {}
     clusterData = {

ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py

@@ -160,6 +160,43 @@ class TestHDP21StackAdvisor(TestCase):
     self.stackAdvisor.recommendHiveConfigurations(configurations, clusterData, None, None)
     self.assertEquals(configurations, expected)
 
+  def test_recommendHbaseEnvConfigurations(self):
+    servicesList = ["HBASE"]
+    configurations = {}
+    components = []
+    hosts = {
+      "items" : [
+        {
+          "Hosts" : {
+            "cpu_count" : 6,
+            "total_mem" : 50331648,
+            "disk_info" : [
+              {"mountpoint" : "/"},
+              {"mountpoint" : "/dev/shm"},
+              {"mountpoint" : "/vagrant"},
+              {"mountpoint" : "/"},
+              {"mountpoint" : "/dev/shm"},
+              {"mountpoint" : "/vagrant"}
+            ]
+          }
+        }
+      ]
+    }
+    expected = {
+      "hbase-env": {
+        "properties": {
+          "hbase_master_heapsize": "8192",
+          "hbase_regionserver_heapsize": "8192",
+          }
+      }
+    }
+
+    clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components)
+    self.assertEquals(clusterData['hbaseRam'], 8)
+
+    self.stackAdvisor.recommendHbaseEnvConfigurations(configurations, clusterData, None, None)
+    self.assertEquals(configurations, expected)
+
   def test_recommendHDFSConfigurations(self):
     configurations = {}
     clusterData = {

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -576,6 +576,43 @@ class TestHDP22StackAdvisor(TestCase):
     }
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
+    
+  def test_recommendHbaseEnvConfigurations(self):
+    servicesList = ["HBASE"]
+    configurations = {}
+    components = []
+    hosts = {
+      "items" : [
+        {
+          "Hosts" : {
+            "cpu_count" : 6,
+            "total_mem" : 50331648,
+            "disk_info" : [
+              {"mountpoint" : "/"},
+              {"mountpoint" : "/dev/shm"},
+              {"mountpoint" : "/vagrant"},
+              {"mountpoint" : "/"},
+              {"mountpoint" : "/dev/shm"},
+              {"mountpoint" : "/vagrant"}
+            ]
+          }
+        }
+      ]
+    }
+    expected = {
+      "hbase-env": {
+        "properties": {
+          "hbase_master_heapsize": "8192",
+          "hbase_regionserver_heapsize": "8192",
+          }
+      }
+    }
+
+    clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components)
+    self.assertEquals(clusterData['hbaseRam'], 8)
+
+    self.stackAdvisor.recommendHbaseEnvConfigurations(configurations, clusterData, None, None)
+    self.assertEquals(configurations, expected)
 
   def test_recommendHDFSConfigurations(self):
     configurations = {}
@@ -631,4 +668,4 @@ class TestHDP22StackAdvisor(TestCase):
                      'type': 'configuration'}]
 
     res = self.stackAdvisor.validateHDFSConfigurationsEnv(properties, recommendedDefaults, configurations, '', '')
-    self.assertEquals(res, res_expected)
+    self.assertEquals(res, res_expected)