AMBARI-6484. HBase RegionServer -Xmn must be configurable (azwaw via aonishuk)

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/configuration/hbase-env.xml

@@ -36,6 +36,16 @@
     <value>1024</value>
     <description>HBase RegionServer Heap Size.</description>
   </property>
+  <property>
+    <name>hbase_regionserver_xmn_max</name>
+    <value>512</value>
+    <description>HBase RegionServer maximum value for minimum heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_ratio</name>
+    <value>0.2</value>
+    <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
+  </property>
   <property>
     <name>hbase_master_heapsize</name>
     <value>1024</value>

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/params.py

@@ -51,7 +51,9 @@ log_dir = config['configurations']['hbase-env']['hbase_log_dir']
 master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
 
 regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
-regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
+regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
+regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
+regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
 
 pid_dir = status_params.pid_dir
 tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/configuration/hbase-env.xml

@@ -36,6 +36,16 @@
     <value>1024</value>
     <description>HBase RegionServer Heap Size.</description>
   </property>
+  <property>
+    <name>hbase_regionserver_xmn_max</name>
+    <value>512</value>
+    <description>HBase RegionServer maximum value for minimum heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_ratio</name>
+    <value>0.2</value>
+    <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
+  </property>
   <property>
     <name>hbase_master_heapsize</name>
     <value>1024</value>

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/params.py

@@ -50,7 +50,9 @@ log_dir = config['configurations']['hbase-env']['hbase_log_dir']
 master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
 
 regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
-regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, 0.2, 512)
+regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
+regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
+regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)
 
 pid_dir = status_params.pid_dir
 tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']

ambari-server/src/test/java/org/apache/ambari/server/api/util/StackExtensionHelperTest.java

@@ -180,8 +180,10 @@ public class StackExtensionHelperTest {
           }
         }
         assertEquals(28, emptyValueProperties.size());
-        assertEquals(66, properties.size());
-        boolean found = false;
+        assertEquals(68, properties.size());
+        boolean foundHBaseClusterDistributed = false;
+        boolean foundHBaseRegionServerXmnMax = false;
+        boolean foundHBaseRegionServerXmnRatio = false;
         for (PropertyInfo property : properties) {
           if (property.getName().equals("hbase.cluster.distributed")) {
             assertEquals("true",
@@ -189,10 +191,27 @@ public class StackExtensionHelperTest {
             assertTrue(property.getDescription().startsWith("The mode the"));
             assertEquals("hbase-site.xml",
                     property.getFilename());
-            found = true;
+            foundHBaseClusterDistributed = true;
+          } else if (property.getName().equals("hbase_regionserver_xmn_max")) {
+            assertEquals("512", property.getValue());
+            assertEquals("global.xml",
+                property.getFilename());
+            foundHBaseRegionServerXmnMax = true;
+          } else if (property.getName().equals("hbase_regionserver_xmn_ratio")) {
+            assertEquals("global.xml",
+                property.getFilename());
+            assertEquals("0.2", property.getValue());
+            foundHBaseRegionServerXmnRatio = true;
           }
         }
-        assertTrue("Property not found in a list of properties", found);
+
+        assertTrue("Property hbase.cluster.distributed not found in a list of properties",
+            foundHBaseClusterDistributed);
+        assertTrue("Property hbase_regionserver_xmn_max not found in a list of properties",
+            foundHBaseRegionServerXmnMax);
+        assertTrue("Property hbase_regionserver_xmn_ratio not found in a list of properties",
+            foundHBaseRegionServerXmnRatio);
+
         List<String> configDependencies = serviceInfo.getConfigDependencies();
         assertEquals(3, configDependencies.size());
         assertEquals("global", configDependencies.get(0));

ambari-server/src/test/python/stacks/1.3.2/configs/secured_no_jce_name.json

@@ -171,7 +171,9 @@
             "hadoop_http_principal_name": "HTTP/_HOST", 
             "kinit_path_local": "/usr/bin", 
             "nagios_keytab_path": "/etc/security/keytabs/nagios.service.keytab", 
-            "hbase_regionserver_heapsize": "1024m", 
+            "hbase_regionserver_heapsize": "1024m",
+            "hbase_regionserver_xmn_max": "512",
+            "hbase_regionserver_xmn_ratio": "0.2",
             "datanode_primary_name": "dn", 
             "namenode_principal_name": "nn/_HOST", 
             "namenode_keytab": "/etc/security/keytabs/nn.service.keytab", 

ambari-server/src/test/python/stacks/2.0.6/configs/flume_target.json

@@ -223,7 +223,9 @@
             "storm_pid_dir": "/var/run/storm", 
             "oozie_pid_dir": "/var/run/oozie", 
             "stormuiserver_host": "c6401.ambari.apache.org", 
-            "hbase_regionserver_heapsize": "1024m", 
+            "hbase_regionserver_heapsize": "1024m",
+            "hbase_regionserver_xmn_max": "512",
+            "hbase_regionserver_xmn_ratio": "0.2",
             "logviewerserver_host": "c6402.ambari.apache.org", 
             "yarn_heapsize": "1024", 
             "hcat_log_dir": "/var/log/webhcat", 

ambari-server/src/test/python/stacks/2.0.6/configs/secured_no_jce_name.json

@@ -236,7 +236,9 @@
             "hadoop_http_principal_name": "HTTP/_HOST", 
             "kinit_path_local": "/usr/bin", 
             "nagios_keytab_path": "/etc/security/keytabs/nagios.service.keytab", 
-            "hbase_regionserver_heapsize": "1024m", 
+            "hbase_regionserver_heapsize": "1024m",
+            "hbase_regionserver_xmn_max": "512",
+            "hbase_regionserver_xmn_ratio": "0.2",
             "resourcemanager_http_primary_name": "HTTP", 
             "datanode_primary_name": "dn", 
             "namenode_principal_name": "nn/_HOST", 

ambari-server/src/test/resources/stacks/HDP/1.3.0/services/HBASE/configuration/global.xml

@@ -49,7 +49,17 @@
   <property>
     <name>hbase_regionserver_heapsize</name>
     <value>1024</value>
-    <description>Log Directories for HBase.</description>
+    <description>RegionServer heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_max</name>
+    <value>512</value>
+    <description>HBase RegionServer maximum value for minimum heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_ratio</name>
+    <value>0.2</value>
+    <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
   </property>
   <property>
     <name>hbase_master_heapsize</name>

ambari-server/src/test/resources/stacks/HDP/2.0.5/services/HBASE/configuration/global.xml

@@ -49,7 +49,17 @@
   <property>
     <name>hbase_regionserver_heapsize</name>
     <value>1024</value>
-    <description>Log Directories for HBase.</description>
+    <description>RegionServers heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_max</name>
+    <value>512</value>
+    <description>HBase RegionServer maximum value for minimum heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_ratio</name>
+    <value>0.2</value>
+    <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
   </property>
   <property>
     <name>hbase_master_heapsize</name>

ambari-server/src/test/resources/stacks/HDP/2.0.7/services/HBASE/configuration/global.xml

@@ -49,7 +49,17 @@
   <property>
     <name>hbase_regionserver_heapsize</name>
     <value>1024</value>
-    <description>Log Directories for HBase.</description>
+    <description>RegionServers heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_max</name>
+    <value>512</value>
+    <description>HBase RegionServer maximum value for minimum heap size.</description>
+  </property>
+  <property>
+    <name>hbase_regionserver_xmn_ratio</name>
+    <value>0.2</value>
+    <description>HBase RegionServer minimum heap size is calculated as a percentage of max heap size.</description>
   </property>
   <property>
     <name>hbase_master_heapsize</name>

ambari-web/app/assets/data/configurations/cluster_level_actual_configs.json

@@ -218,6 +218,8 @@
         "webhcat_user" : "hcat",
         "regionserver_handlers" : "30",
         "hbase_regionserver_heapsize" : "1024m",
+        "hbase_regionserver_xmn_max": "512",
+        "hbase_regionserver_xmn_ratio": "0.2",
         "dfs_include" : "dfs.include",
         "dfs_webhdfs_enabled" : "false",
         "scheduler_name" : "org.apache.hadoop.mapred.CapacityTaskScheduler",

ambari-web/app/assets/data/configurations/cluster_level_configs.json

@@ -533,6 +533,8 @@
         "webhcat_user" : "hcat",
         "regionserver_handlers" : "30",
         "hbase_regionserver_heapsize" : "1024m",
+        "hbase_regionserver_xmn_max": "512",
+        "hbase_regionserver_xmn_ratio": "0.2",
         "dfs_include" : "dfs.include",
         "dfs_webhdfs_enabled" : "false",
         "scheduler_name" : "org.apache.hadoop.mapred.CapacityTaskScheduler",

ambari-web/app/assets/data/wizard/stack/hdp/version1.3.0/HBASE.json

@@ -376,7 +376,7 @@
     {
       "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_heapsize",
       "StackConfigurations" : {
-        "property_description" : "Log Directories for HBase.",
+        "property_description" : "RegionServers heap size.",
         "property_name" : "hbase_regionserver_heapsize",
         "property_value" : "1024",
         "service_name" : "HBASE",
@@ -385,6 +385,30 @@
         "type" : "global.xml"
       }
     },
+    {
+      "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_max",
+      "StackConfigurations" : {
+        "property_description" : "Maximum value for minimum RegionServers Java heap size",
+        "property_name" : "hbase_regionserver_xmn_max",
+        "property_value" : "512",
+        "service_name" : "HBASE",
+        "stack_name" : "HDP",
+        "stack_version" : "1.3.0",
+        "type" : "global.xml"
+      }
+    },
+    {
+      "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_ratio",
+      "StackConfigurations" : {
+        "property_description" : "RegionServers -Xmn is calculated as a percentage of -Xmx",
+        "property_name" : "hbase_regionserver_xmn_ratio",
+        "property_value" : "0.2",
+        "service_name" : "HBASE",
+        "stack_name" : "HDP",
+        "stack_version" : "1.3.0",
+        "type" : "global.xml"
+      }
+    },
     {
       "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_user",
       "StackConfigurations" : {

ambari-web/app/assets/data/wizard/stack/hdp/version1.3.0/global.json

@@ -915,6 +915,26 @@
             "stack_name" : "HDP"
           }
         },
+        {
+          "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_max",
+          "StackConfigurations" : {
+            "stack_version" : "1.3.0",
+            "property_name" : "hbase_regionserver_xmn_max",
+            "filename" : "global.xml",
+            "service_name" : "HBASE",
+            "stack_name" : "HDP"
+          }
+        },
+        {
+          "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_ratio",
+          "StackConfigurations" : {
+            "stack_version" : "1.3.0",
+            "property_name" : "hbase_regionserver_xmn_ratio",
+            "filename" : "global.xml",
+            "service_name" : "HBASE",
+            "stack_name" : "HDP"
+          }
+        },
         {
           "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/client_scannercaching",
           "StackConfigurations" : {

ambari-web/app/assets/data/wizard/stack/hdp/version131/HBASE.json

@@ -148,7 +148,7 @@
     {
       "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_heapsize",
       "StackConfigurations" : {
-        "property_description" : "Log Directories for HBase.",
+        "property_description" : "RegionServers HeapSize.",
         "property_value" : "1024",
         "stack_version" : "1.3.1",
         "property_name" : "hbase_regionserver_heapsize",
@@ -157,6 +157,30 @@
         "stack_name" : "HDP"
       }
     },
+    {
+      "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_max",
+      "StackConfigurations" : {
+        "property_description" : "Maximum value for minimum RegionServers Java heap size",
+        "property_name" : "hbase_regionserver_xmn_max",
+        "property_value" : "512",
+        "service_name" : "HBASE",
+        "stack_name" : "HDP",
+        "stack_version" : "1.3.1",
+        "type" : "global.xml"
+      }
+    },
+    {
+      "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_ratio",
+      "StackConfigurations" : {
+        "property_description" : "RegionServers -Xmn is calculated as a percentage of -Xmx",
+        "property_name" : "hbase_regionserver_xmn_ratio",
+        "property_value" : "0.2",
+        "service_name" : "HBASE",
+        "stack_name" : "HDP",
+        "stack_version" : "1.3.1",
+        "type" : "global.xml"
+      }
+    },
     {
       "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hregion_majorcompaction",
       "StackConfigurations" : {

ambari-web/app/assets/data/wizard/stack/hdp/version131/global.json

@@ -915,6 +915,26 @@
             "stack_name" : "HDP"
           }
         },
+        {
+          "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_max",
+          "StackConfigurations" : {
+            "stack_version" : "1.3.1",
+            "property_name" : "hbase_regionserver_xmn_max",
+            "filename" : "global.xml",
+            "service_name" : "HBASE",
+            "stack_name" : "HDP"
+          }
+        },
+        {
+          "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_ratio",
+          "StackConfigurations" : {
+            "stack_version" : "1.3.1",
+            "property_name" : "hbase_regionserver_xmn_ratio",
+            "filename" : "global.xml",
+            "service_name" : "HBASE",
+            "stack_name" : "HDP"
+          }
+        },
         {
           "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/client_scannercaching",
           "StackConfigurations" : {

ambari-web/app/assets/data/wizard/stack/hdp/version2.0.1/HBASE.json

@@ -376,7 +376,7 @@
     {
       "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/2.0.1/stackServices/HBASE/configurations/hbase_regionserver_heapsize",
       "StackConfigurations" : {
-        "property_description" : "Log Directories for HBase.",
+        "property_description" : "RegionServers heap size.",
         "property_name" : "hbase_regionserver_heapsize",
         "property_value" : "1024",
         "service_name" : "HBASE",
@@ -385,6 +385,30 @@
         "type" : "global.xml"
       }
     },
+    {
+      "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/2.0.1/stackServices/HBASE/configurations/hbase_regionserver_xmn_max",
+      "StackConfigurations" : {
+        "property_description" : "Maximum value for minimum RegionServers Java heap size",
+        "property_name" : "hbase_regionserver_xmn_max",
+        "property_value" : "512",
+        "service_name" : "HBASE",
+        "stack_name" : "HDP",
+        "stack_version" : "2.0.1",
+        "type" : "global.xml"
+      }
+    },
+    {
+      "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/2.0.1/stackServices/HBASE/configurations/hbase_regionserver_xmn_ratio",
+      "StackConfigurations" : {
+        "property_description" : "RegionServers -Xmn is calculated as a percentage of -Xmx",
+        "property_name" : "hbase_regionserver_xmn_ratio",
+        "property_value" : "0.2",
+        "service_name" : "HBASE",
+        "stack_name" : "HDP",
+        "stack_version" : "2.0.1",
+        "type" : "global.xml"
+      }
+    },
     {
       "href" : "http://192.168.56.101:8080/api/v1/stacks2/HDP/versions/2.0.1/stackServices/HBASE/configurations/hbase_user",
       "StackConfigurations" : {

ambari-web/app/assets/data/wizard/stack/hdp/version2.0.1/global.json

@@ -915,6 +915,26 @@
             "stack_name" : "HDP"
           }
         },
+        {
+          "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_max",
+          "StackConfigurations" : {
+            "stack_version" : "1.3.0",
+            "property_name" : "hbase_regionserver_xmn_max",
+            "filename" : "global.xml",
+            "service_name" : "HBASE",
+            "stack_name" : "HDP"
+          }
+        },
+        {
+          "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/hbase_regionserver_xmn_ratio",
+          "StackConfigurations" : {
+            "stack_version" : "1.3.0",
+            "property_name" : "hbase_regionserver_xmn_ratio",
+            "filename" : "global.xml",
+            "service_name" : "HBASE",
+            "stack_name" : "HDP"
+          }
+        },
         {
           "href" : "http://dev.hortonworks.com:8080/api/v1/stacks2/HDP/versions/1.3.0/stackServices/HBASE/configurations/client_scannercaching",
           "StackConfigurations" : {

ambari-web/app/data/HDP2/site_properties.js

@@ -2072,6 +2072,27 @@ module.exports =
       "category": "HBASE_REGIONSERVER",
       "index": 1
     },
+    {
+     "id": "puppet var",
+     "name": "hbase_regionserver_xmn_max",
+     "displayName": "RegionServers maximum value for -Xmn",
+     "displayType": "int",
+     "unit": "MB",
+     "isVisible": true,
+     "serviceName": "HBASE",
+     "category": "HBASE_REGIONSERVER",
+     "index": 6
+    },
+    {
+      "id": "puppet var",
+      "name": "hbase_regionserver_xmn_ratio",
+      "displayName": "RegionServers -Xmn in -Xmx ratio",
+      "displayType": "float",
+      "isVisible": true,
+      "serviceName": "HBASE",
+      "category": "HBASE_REGIONSERVER",
+      "index": 7
+    },
     {
       "id": "puppet var",
       "name": "hbase_log_dir",

ambari-web/app/data/site_properties.js

@@ -1340,6 +1340,29 @@ module.exports =
       "category": "HBASE_REGIONSERVER",
       "index": 1
     },
+    {
+      "id": "puppet var",
+      "name": "hbase_regionserver_xmn_max",
+      "displayName": "RegionServers maximum value for -Xmn",
+      "displayType": "int",
+      "unit": "MB",
+      "isVisible": true,
+      "domain": "regionserver-global",
+      "serviceName": "HBASE",
+      "category": "HBASE_REGIONSERVER",
+      "index": 6
+    },
+    {
+      "id": "puppet var",
+      "name": "hbase_regionserver_xmn_ratio",
+      "displayName": "RegionServers Xmn is in percentage of Xmx",
+      "displayType": "float",
+      "isVisible": true,
+      "domain": "regionserver-global",
+      "serviceName": "HBASE",
+      "category": "HBASE_REGIONSERVER",
+      "index": 7
+    },
     {
       "id": "puppet var",
       "name": "hbase_log_dir",

contrib/views/slider/src/main/resources/ui/app/assets/data/apps/apps.json

@@ -148,6 +148,8 @@
           "ganglia_server_port" : "8663",
           "hbase_master_heapsize" : "1024m",
           "hbase_regionserver_heapsize" : "1024m",
+          "hbase_regionserver_xmn_size" : "512",
+          "hbase_regionserver_xmn_ratio" : "0.2",
           "security_enabled" : "false",
           "user_group" : "hadoop"
         },

contrib/views/slider/src/main/resources/ui/app/assets/data/apptypes/all_fields.json

@@ -425,6 +425,8 @@
         "site.global.ganglia_server_port" : "8667",
         "site.global.hbase_master_heapsize" : "1024m",
         "site.global.hbase_regionserver_heapsize" : "1024m",
+        "site.global.hbase_regionserver_xmn_size" : "512",
+        "site.global.hbase_regionserver_xmn_ratio" : "0.2",
         "site.global.security_enabled" : "false",
         "site.global.user_group" : "hadoop",
         "site.hbase-site.hbase.client.keyvalue.maxsize" : "10485760",