AMBARI-10111. Tune GC settings for HBase daemon. (swagle)

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml

@@ -34,22 +34,44 @@
   <property>
     <name>hbase_regionserver_heapsize</name>
     <value>1024m</value>
-    <description>HBase RegionServer Heap Size.</description>
+    <description>
+        HBase RegionServer Heap Size. In embedded mode, total heap size is
+        sum of master and regionserver heap sizes.
+    </description>
   </property>
   <property>
-    <name>hbase_regionserver_xmn_max</name>
-    <value>512m</value>
-    <description>HBase RegionServer maximum value for minimum heap size.</description>
+    <name>regionserver_xmn_size</name>
+    <value>256m</value>
+    <description>HBase RegionServer maximum value for young generation heap size.</description>
+  </property>
+  <property>
+    <name>hbase_master_xmn_size</name>
+    <value>256m</value>
+    <description>
+      HBase Master maximum value for young generation heap size.
+    </description>
+  </property>
+  <property>
+    <name>hbase_master_maxperm_size</name>
+    <value>128m</value>
+    <description>HBase RegionServer maximum value for perm heap size.</description>
   </property>
   <property>
     <name>hbase_regionserver_xmn_ratio</name>
     <value>0.2</value>
-    <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
+    <description>
+        HBase RegionServer minimum heap size is calculated as a percentage
+        of max heap size. If hbase_regionserver_xmn_max is defined, the ratio
+        will be used to calculate -Xmn setting.
+    </description>
   </property>
   <property>
     <name>hbase_master_heapsize</name>
     <value>1024m</value>
-    <description>HBase Master Heap Size</description>
+    <description>
+        HBase Master Heap Size. In embedded mode, total heap size is
+        sum of master and regionserver heap sizes.
+    </description>
   </property>
 
   <!-- hbase-env.sh -->
@@ -85,8 +107,8 @@ export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -X
 # More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
 #
 # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
-export HBASE_MASTER_OPTS="-Xmx{{master_heapsize}}"
-export HBASE_REGIONSERVER_OPTS="-Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70  -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
+export HBASE_MASTER_OPTS=" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}} -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
+export HBASE_REGIONSERVER_OPTS="-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
 # export HBASE_THRIFT_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
 # export HBASE_ZOOKEEPER_OPTS="$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/hbase.py

@@ -59,17 +59,6 @@ def hbase(name=None):
           owner = params.hadoop_user
     )
 
-  # File(format("{hbase_conf_dir}/hbase-env.cmd"),
-  #      owner = params.hadoop_user,
-  #      content=InlineTemplate(params.hbase_env_sh_template)
-  # )
-
-  # Metrics properties
-  # File(os.path.join(params.hbase_conf_dir, "hadoop-metrics2-hbase.properties"),
-  #      owner = params.hadoop_user,
-  #      content=Template("hadoop-metrics2-hbase.properties.j2")
-  # )
-
   hbase_TemplateConfig('regionservers', user=params.hadoop_user)
 
   if params.security_enabled:

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -31,6 +31,15 @@ else:
 config = Script.get_config()
 exec_tmp_dir = Script.get_tmp_dir()
 
+def get_combined_memory_mb(value1, value2):
+  try:
+    part1 = int(value1.strip()[:-1]) if value1.lower().strip()[-1:] == 'm' else int(value1)
+    part2 = int(value2.strip()[:-1]) if value2.lower().strip()[-1:] == 'm' else int(value2)
+    return str(part1 + part2) + 'm'
+  except:
+    return None
+pass
+
 #AMBARI_METRICS data
 ams_pid_dir = status_params.ams_collector_pid_dir
 
@@ -77,13 +86,26 @@ metrics_collector_heapsize = default('/configurations/ams-env/metrics_collector_
 
 hbase_log_dir = config['configurations']['ams-hbase-env']['hbase_log_dir']
 master_heapsize = config['configurations']['ams-hbase-env']['hbase_master_heapsize']
-
 regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regionserver_heapsize']
-regionserver_xmn_max = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_max']
-regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio']
-regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
-# For embedded mode
-hbase_heapsize = master_heapsize
+
+regionserver_xmn_max = default('configurations/ams-hbase-env/hbase_regionserver_xmn_max', None)
+if regionserver_xmn_max:
+  regionserver_xmn_percent = config['configurations']['ams-hbase-env']['hbase_regionserver_xmn_ratio']
+  regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
+else:
+  regionserver_xmn_size = config['configurations']['ams-hbase-env']['regionserver_xmn_size']
+pass
+
+hbase_master_xmn_size = config['configurations']['ams-hbase-env']['hbase_master_xmn_size']
+hbase_master_maxperm_size = config['configurations']['ams-hbase-env']['hbase_master_maxperm_size']
+
+# Choose heap size for embedded mode as sum of master + regionserver
+if not is_hbase_distributed:
+  hbase_heapsize = get_combined_memory_mb(master_heapsize, regionserver_heapsize)
+  if hbase_heapsize is None:
+    hbase_heapsize = master_heapsize
+else:
+  hbase_heapsize = master_heapsize
 
 zookeeper_quorum_hosts = ','.join(ams_collector_hosts) if is_hbase_distributed else 'localhost'
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -148,7 +148,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
       totalHostsCount = len(hosts["items"])
       # blockCache = 0.3, memstore = 0.3, phoenix-server = 0.2, phoenix-client = 0.3
       if totalHostsCount >= 400:
-        putHbaseEnvProperty("hbase_master_heapsize", "12288m")
+        putHbaseEnvProperty("hbase_regionserver_heapsize", "12288m")
         putAmsEnvProperty("metrics_collector_heapsize", "8192m")
         putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
         putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
@@ -158,19 +158,31 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.25)
         putAmsHbaseSiteProperty("phoenix.query.maxGlobalMemoryPercentage", 20)
         putTimelineServiceProperty("phoenix.query.maxGlobalMemoryPercentage", 30)
+        putAmsHbaseSiteProperty("hbase_master_xmn_size", "512m")
+        putAmsHbaseSiteProperty("regionserver_xmn_size", "512m")
       elif totalHostsCount >= 100:
-        putHbaseEnvProperty("hbase_master_heapsize", "6144m")
+        putHbaseEnvProperty("hbase_regionserver_heapsize", "6144m")
         putAmsEnvProperty("metrics_collector_heapsize", "4096m")
         putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
         putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
         putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
         putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
+        putAmsHbaseSiteProperty("hbase_master_xmn_size", "512m")
       elif totalHostsCount >= 50:
-        putHbaseEnvProperty("hbase_master_heapsize", "2048m")
+        putHbaseEnvProperty("hbase_regionserver_heapsize", "2048m")
+        putHbaseEnvProperty("hbase_master_heapsize", "512m")
         putAmsEnvProperty("metrics_collector_heapsize", "2048m")
+        putAmsHbaseSiteProperty("hbase_master_xmn_size", "256m")
       else:
-        putHbaseEnvProperty("hbase_master_heapsize", "1024m")
+        # Embedded mode heap size : master + regionserver
+        putHbaseEnvProperty("hbase_regionserver_heapsize", "512m")
+        putHbaseEnvProperty("hbase_master_heapsize", "512m")
         putAmsEnvProperty("metrics_collector_heapsize", "512m")
+        putAmsHbaseSiteProperty("hbase_master_xmn_size", "128m")
+      pass
+    pass
+
+
 
   def getConfigurationClusterSummary(self, servicesList, hosts, components):
 
@@ -377,7 +389,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
                 masterHostMessage.format(
                   collectorHostName, str(", ".join(hostMasterComponents[collectorHostName]))))
 
-            # No enough physical memory
+            # Not enough physical memory
             # TODO Add AMBARI_METRICS Collector Xmx property to ams-env
             requiredMemory = getMemorySizeRequired(hostComponents[collectorHostName], configurations)
             if host["Hosts"]["total_mem"] * 1024 < requiredMemory:  # in bytes
@@ -388,8 +400,17 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
               regionServerItem = self.getErrorItem(message)
               masterItem = self.getErrorItem(message)
               break
+      pass
+
+    # Check RS memory in distributed mode since we set default as 512m
+    hbase_site = getSiteProperties(configurations, "ams-hbase-site")
+    hbase_rootdir = hbase_site.get("hbase.rootdir")
+    regionServerMinMemItem = None
+    if hbase_rootdir.startswith("hdfs://"):
+      regionServerMinMemItem = self.validateMinMemorySetting(properties, 1024, 'hbase_regionserver_heapsize')
 
     validationItems = [{"config-name": "hbase_regionserver_heapsize", "item": regionServerItem},
+                       {"config-name": "hbase_regionserver_heapsize", "item": regionServerMinMemItem},
                        {"config-name": "hbase_master_heapsize", "item": masterItem},
                        {"config-name": "hbase_master_heapsize", "item": masterHostItem},
                        {"config-name": "hbase_log_dir", "item": logDirItem}]
@@ -469,6 +490,27 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     return None
 
+  def validateMinMemorySetting(self, properties, defaultValue, propertyName):
+    if not propertyName in properties:
+      return self.getErrorItem("Value should be set")
+    if defaultValue is None:
+      return self.getErrorItem("Config's default value can't be null or undefined")
+
+    value = properties[propertyName]
+    if value is None:
+      return self.getErrorItem("Value can't be null or undefined")
+    try:
+      valueInt = int(value.strip()[:-1])
+      # TODO: generify for other use cases
+      defaultValueInt = int(str(defaultValue).strip())
+      if valueInt < defaultValueInt:
+        return self.getWarnItem("Value is less than the minimum recommended default of -Xmx" + str(defaultValue))
+    except:
+      return None
+
+    return None
+
+
   def validateXmxValue(self, properties, recommendedDefaults, propertyName):
     if not propertyName in properties:
       return self.getErrorItem("Value should be set")

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -673,8 +673,8 @@ class TestHDP22StackAdvisor(TestCase):
     expected = {
       "ams-hbase-env": {
         "properties": {
-          "hbase_master_heapsize": "1024m",
-          "hbase_regionserver_heapsize": "1024m",
+          "hbase_master_heapsize": "512m",
+          "hbase_regionserver_heapsize": "512m",
           }
       },
       "ams-env": {
@@ -686,7 +686,8 @@ class TestHDP22StackAdvisor(TestCase):
         "properties": {
           "hbase.regionserver.global.memstore.lowerLimit": "0.3",
           "hbase.regionserver.global.memstore.upperLimit": "0.35",
-          "hfile.block.cache.size": "0.3"
+          "hfile.block.cache.size": "0.3",
+          "hbase_master_xmn_size" : "128m"
         }
       },
       "ams-site": {