AMBARI-15428. Fix HBase master heap recommendations.(vbrodetskyi)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -281,9 +281,25 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
   def recommendHbaseConfigurations(self, configurations, clusterData, services, hosts):
     # recommendations for HBase env config
+
+    # If cluster size is < 100, hbase master heap = 2G
+    # else If cluster size is < 500, hbase master heap = 4G
+    # else hbase master heap = 8G
+    # for small test clusters use 1 gb
+    hostsCount = 0
+    if hosts and "items" in hosts:
+      hostsCount = len(hosts["items"])
+
+    hbaseMasterRam = {
+      hostsCount < 20: 1,
+      20 <= hostsCount < 100: 2,
+      100 <= hostsCount < 500: 4,
+      500 <= hostsCount: 8
+    }[True]
+
     putHbaseProperty = self.putProperty(configurations, "hbase-env", services)
     putHbaseProperty('hbase_regionserver_heapsize', int(clusterData['hbaseRam']) * 1024)
-    putHbaseProperty('hbase_master_heapsize', int(clusterData['hbaseRam']) * 1024)
+    putHbaseProperty('hbase_master_heapsize', hbaseMasterRam * 1024)
 
     # recommendations for HBase site config
     putHbaseSiteProperty = self.putProperty(configurations, "hbase-site", services)
@@ -782,7 +798,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       {"os":12, "hbase":16},
       {"os":24, "hbase":24},
       {"os":32, "hbase":32},
-      {"os":64, "hbase":64}
+      {"os":64, "hbase":32}
     ]
     index = {
       cluster["ram"] <= 4: 0,
@@ -797,9 +813,12 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       128 < cluster["ram"] <= 256: 9,
       256 < cluster["ram"]: 10
     }[1]
+
+
     cluster["reservedRam"] = ramRecommendations[index]["os"]
     cluster["hbaseRam"] = ramRecommendations[index]["hbase"]
 
+
     cluster["minContainerSize"] = {
       cluster["ram"] <= 4: 256,
       4 < cluster["ram"] <= 8: 512,

ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py

@@ -665,23 +665,22 @@ class TestHDP206StackAdvisor(TestCase):
     servicesList = ["HBASE"]
     configurations = {}
     components = []
+    host_item = {
+      "Hosts" : {
+        "cpu_count" : 6,
+        "total_mem" : 50331648,
+        "disk_info" : [
+          {"mountpoint" : "/"},
+          {"mountpoint" : "/dev/shm"},
+          {"mountpoint" : "/vagrant"},
+          {"mountpoint" : "/"},
+          {"mountpoint" : "/dev/shm"},
+          {"mountpoint" : "/vagrant"}
+        ]
+      }
+    }
     hosts = {
-      "items" : [
-        {
-          "Hosts" : {
-            "cpu_count" : 6,
-            "total_mem" : 50331648,
-            "disk_info" : [
-              {"mountpoint" : "/"},
-              {"mountpoint" : "/dev/shm"},
-              {"mountpoint" : "/vagrant"},
-              {"mountpoint" : "/"},
-              {"mountpoint" : "/dev/shm"},
-              {"mountpoint" : "/vagrant"}
-            ]
-          }
-        }
-      ]
+      "items" : [host_item for i in range(1, 300)]
     }
     services = {
       "services" : [
@@ -707,7 +706,7 @@ class TestHDP206StackAdvisor(TestCase):
       },
       "hbase-env": {
         "properties": {
-          "hbase_master_heapsize": "8192",
+          "hbase_master_heapsize": "4096",
           "hbase_regionserver_heapsize": "8192",
           }
       }
@@ -716,7 +715,7 @@ class TestHDP206StackAdvisor(TestCase):
     clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, None)
     self.assertEquals(clusterData['hbaseRam'], 8)
 
-    self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, None)
+    self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
 

ambari-server/src/test/python/stacks/2.1/common/test_stack_advisor.py

@@ -283,23 +283,22 @@ class TestHDP21StackAdvisor(TestCase):
     servicesList = ["HBASE"]
     configurations = {}
     components = []
+    host_item = {
+      "Hosts" : {
+        "cpu_count" : 6,
+        "total_mem" : 50331648,
+        "disk_info" : [
+          {"mountpoint" : "/"},
+          {"mountpoint" : "/dev/shm"},
+          {"mountpoint" : "/vagrant"},
+          {"mountpoint" : "/"},
+          {"mountpoint" : "/dev/shm"},
+          {"mountpoint" : "/vagrant"}
+        ]
+      }
+    }
     hosts = {
-      "items" : [
-        {
-          "Hosts" : {
-            "cpu_count" : 6,
-            "total_mem" : 50331648,
-            "disk_info" : [
-              {"mountpoint" : "/"},
-              {"mountpoint" : "/dev/shm"},
-              {"mountpoint" : "/vagrant"},
-              {"mountpoint" : "/"},
-              {"mountpoint" : "/dev/shm"},
-              {"mountpoint" : "/vagrant"}
-            ]
-          }
-        }
-      ]
+      "items" : [host_item for i in range(1, 600)]
     }
     services = {
       "services" : [
@@ -334,7 +333,7 @@ class TestHDP21StackAdvisor(TestCase):
     clusterData = self.stackAdvisor.getConfigurationClusterSummary(servicesList, hosts, components, None)
     self.assertEquals(clusterData['hbaseRam'], 8)
 
-    self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, None)
+    self.stackAdvisor.recommendHbaseConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
   def test_recommendHDFSConfigurations(self):

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -2305,7 +2305,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "hbase-env": {
         "properties": {
-          "hbase_master_heapsize": "8192",
+          "hbase_master_heapsize": "1024",
           "hbase_regionserver_heapsize": "8192",
           }
       }
@@ -2542,7 +2542,7 @@ class TestHDP22StackAdvisor(TestCase):
       },
       "hbase-env": {
         "properties": {
-          "hbase_master_heapsize": "8192",
+          "hbase_master_heapsize": "1024",
           "hbase_regionserver_heapsize": "8192",
         },
         "property_attributes": {

ambari-server/src/test/python/stacks/2.3/common/test_stack_advisor.py

@@ -857,7 +857,7 @@ class TestHDP23StackAdvisor(TestCase):
       },
       "hbase-env": {
         "properties": {
-          "hbase_master_heapsize": "114688",
+          "hbase_master_heapsize": "1024",
           "hbase_max_direct_memory_size": "94208",
           "hbase_regionserver_heapsize": "20480"
         }
@@ -987,7 +987,7 @@ class TestHDP23StackAdvisor(TestCase):
     expected["hbase-site"]["property_attributes"]["hbase.bucketcache.ioengine"] = {"delete": "true"}
     expected["hbase-site"]["property_attributes"]["hbase.bucketcache.percentage.in.combinedcache"] = {"delete": "true"}
     expected["hbase-env"]["property_attributes"] = {"hbase_max_direct_memory_size" : {"delete": "true"}}
-    expected["hbase-env"]["properties"]["hbase_master_heapsize"] = "4096"
+    expected["hbase-env"]["properties"]["hbase_master_heapsize"] = "1024"
     expected["hbase-env"]["properties"]["hbase_regionserver_heapsize"] = "4096"
     self.stackAdvisor.recommendHBASEConfigurations(configurations, clusterData, services, None)
     self.assertEquals(configurations, expected)