AMBARI-13223. AMS Collector does not start (dlysnichenko)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java

@@ -39,6 +39,9 @@ import java.util.Map;
 public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
 
   private static final String STORM_SITE = "storm-site";
+  private static final String AMS_ENV = "ams-env";
+  private static final String AMS_HBASE_ENV = "ams-hbase-env";
+
 
   /**
    * Logger.
@@ -100,6 +103,7 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
   @Override
   protected void executeDMLUpdates() throws AmbariException, SQLException {
     addMissingConfigs();
+    updateAMSConfigs();
   }
 
   protected void addMissingConfigs() throws AmbariException {
@@ -130,4 +134,110 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
       }
     }
   }
+
+  protected void updateAMSConfigs() throws AmbariException {
+    AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
+    Clusters clusters = ambariManagementController.getClusters();
+
+    if (clusters != null) {
+      Map<String, Cluster> clusterMap = clusters.getClusters();
+
+      if (clusterMap != null && !clusterMap.isEmpty()) {
+        for (final Cluster cluster : clusterMap.values()) {
+          Config amsEnv = cluster.getDesiredConfigByType(AMS_ENV);
+          if (amsEnv != null) {
+            Map<String, String> amsEnvProperties = amsEnv.getProperties();
+
+            String metrics_collector_heapsize = amsEnvProperties.get("metrics_collector_heapsize");
+            String content = amsEnvProperties.get("content");
+            Map<String, String> newProperties = new HashMap<>();
+            newProperties.put("metrics_collector_heapsize", memoryToIntMb(metrics_collector_heapsize));
+            newProperties.put("content", updateAmsEnvContent(content));
+            updateConfigurationPropertiesForCluster(cluster, AMS_ENV, newProperties, true, true);
+          }
+          Config amsHbaseEnv = cluster.getDesiredConfigByType(AMS_HBASE_ENV);
+          if (amsHbaseEnv != null) {
+            Map<String, String> amsHbaseEnvProperties = amsHbaseEnv.getProperties();
+            String hbase_regionserver_heapsize = amsHbaseEnvProperties.get("hbase_regionserver_heapsize");
+            String regionserver_xmn_size = amsHbaseEnvProperties.get("regionserver_xmn_size");
+            String hbase_master_xmn_size = amsHbaseEnvProperties.get("hbase_master_xmn_size");
+            String hbase_master_maxperm_size = amsHbaseEnvProperties.get("hbase_master_maxperm_size");
+            String hbase_master_heapsize = amsHbaseEnvProperties.get("hbase_master_heapsize");
+            String content = amsHbaseEnvProperties.get("content");
+
+            Map<String, String> newProperties = new HashMap<>();
+            newProperties.put("hbase_regionserver_heapsize", memoryToIntMb(hbase_regionserver_heapsize));
+            newProperties.put("regionserver_xmn_size", memoryToIntMb(regionserver_xmn_size));
+            newProperties.put("hbase_master_xmn_size", memoryToIntMb(hbase_master_xmn_size));
+            newProperties.put("hbase_master_maxperm_size", memoryToIntMb(hbase_master_maxperm_size));
+            newProperties.put("hbase_master_heapsize", memoryToIntMb(hbase_master_heapsize));
+            newProperties.put("content", updateAmsHbaseEnvContent(content));
+            updateConfigurationPropertiesForCluster(cluster, AMS_HBASE_ENV, newProperties, true, true);
+          }
+        }
+      }
+    }
+
+  }
+
+  protected String updateAmsEnvContent(String oldContent) {
+    if (oldContent == null) {
+      return null;
+    }
+    String regSearch = "export\\s*AMS_COLLECTOR_HEAPSIZE\\s*=\\s*\\{\\{metrics_collector_heapsize\\}\\}";
+    String replacement = "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}m";
+    return oldContent.replaceAll(regSearch, replacement);
+  }
+
+  protected String updateAmsHbaseEnvContent(String content) {
+    if (content == null) {
+      return null;
+    }
+
+    String regSearch = "\\{\\{hbase_heapsize\\}\\}";
+    String replacement = "{{hbase_heapsize}}m";
+    content = content.replaceAll(regSearch, replacement);
+    regSearch = "\\{\\{hbase_master_maxperm_size\\}\\}";
+    replacement = "{{hbase_master_maxperm_size}}m";
+    content = content.replaceAll(regSearch, replacement);
+    regSearch = "\\{\\{hbase_master_xmn_size\\}\\}";
+    replacement = "{{hbase_master_xmn_size}}m";
+    content = content.replaceAll(regSearch, replacement);
+    regSearch = "\\{\\{regionserver_xmn_size\\}\\}";
+    replacement = "{{regionserver_xmn_size}}m";
+    content = content.replaceAll(regSearch, replacement);
+    regSearch = "\\{\\{regionserver_heapsize\\}\\}";
+    replacement = "{{regionserver_heapsize}}m";
+    content = content.replaceAll(regSearch, replacement);
+    return content;
+  }
+
+  private String memoryToIntMb(String memorySize) {
+    if (memorySize == null) {
+      return "0";
+    }
+    Integer value = 0;
+    try {
+      value = Integer.parseInt(memorySize.replaceAll("\\D+", ""));
+    } catch (NumberFormatException ex) {
+      LOG.error(ex.getMessage());
+    }
+    char unit = memorySize.toUpperCase().charAt(memorySize.length() - 1);
+    // Recalculate memory size to Mb
+    switch (unit) {
+      case 'K':
+        value /= 1024;
+        break;
+      case 'B':
+        value /= (1024*1024);
+        break;
+      case 'G':
+        value *= 1024;
+        break;
+      case 'T':
+        value *= 1024*1024;
+        break;
+    }
+    return value.toString();
+  }
 }

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml

@@ -51,8 +51,12 @@
   </property>
   <property>
     <name>metrics_collector_heapsize</name>
-    <value>512m</value>
+    <value>512</value>
     <description>Metrics Collector Heap Size</description>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+    </value-attributes>
   </property>
   <property>
     <name>content</name>
@@ -78,7 +82,7 @@ export AMS_MONITOR_PID_DIR={{ams_monitor_pid_dir}}
 export AMS_HBASE_PID_DIR={{hbase_pid_dir}}
 
 # AMS Collector heapsize
-export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}
+export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}m
 
 # AMS Collector options
 export AMS_COLLECTOR_OPTS="-Djava.library.path=/usr/lib/ams-hbase/lib/hadoop-native"

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml

@@ -33,11 +33,15 @@
   </property>
   <property>
     <name>hbase_regionserver_heapsize</name>
-    <value>512m</value>
+    <value>512</value>
     <description>
         HBase RegionServer Heap Size. In embedded mode, total heap size is
         sum of master and regionserver heap sizes.
     </description>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+    </value-attributes>
     <depends-on>
       <property>
         <type>ams-hbase-site</type>
@@ -51,20 +55,32 @@
   </property>
   <property>
     <name>regionserver_xmn_size</name>
-    <value>256m</value>
+    <value>256</value>
     <description>HBase RegionServer maximum value for young generation heap size.</description>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+    </value-attributes>
   </property>
   <property>
     <name>hbase_master_xmn_size</name>
-    <value>256m</value>
+    <value>256</value>
     <description>
       HBase Master maximum value for young generation heap size.
     </description>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+    </value-attributes>
   </property>
   <property>
     <name>hbase_master_maxperm_size</name>
-    <value>128m</value>
+    <value>128</value>
     <description>HBase RegionServer maximum value for perm heap size.</description>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+    </value-attributes>
   </property>
   <property>
     <name>hbase_regionserver_xmn_ratio</name>
@@ -73,11 +89,15 @@
   </property>
   <property>
     <name>hbase_master_heapsize</name>
-    <value>512m</value>
+    <value>512</value>
     <description>
         HBase Master Heap Size. In embedded mode, total heap size is
         sum of master and regionserver heap sizes.
     </description>
+    <value-attributes>
+      <type>int</type>
+      <unit>MB</unit>
+    </value-attributes>
     <depends-on>
       <property>
         <type>ams-hbase-site</type>
@@ -114,7 +134,7 @@ export HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}
 export HBASE_CLASSPATH=${HBASE_CLASSPATH}
 
 # The maximum amount of heap to use, in MB. Default is 1000.
-export HBASE_HEAPSIZE={{hbase_heapsize}}
+export HBASE_HEAPSIZE={{hbase_heapsize}}m
 
 # Extra Java runtime options.
 # Below are what we set by default. May only work with SUN JVM.
@@ -132,11 +152,11 @@ export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -X
 # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
 
 {% if java_version &lt; 8 %}
-export HBASE_MASTER_OPTS=" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}} -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
-export HBASE_REGIONSERVER_OPTS="-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
+export HBASE_MASTER_OPTS=" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}}m -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
+export HBASE_REGIONSERVER_OPTS="-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m"
 {% else %}
-export HBASE_MASTER_OPTS=" -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
-export HBASE_REGIONSERVER_OPTS=" -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
+export HBASE_MASTER_OPTS=" -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
+export HBASE_REGIONSERVER_OPTS=" -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m"
 {% endif %}
 
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/functions.py

@@ -25,16 +25,14 @@ import datetime
 
 from resource_management.core.shell import checked_call
 
-def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
+def calc_xmn_from_xms(heapsize, xmn_percent, xmn_max):
   """
-  @param heapsize_str: str (e.g '1000m')
+  @param heapsize: int (e.g 1000)
   @param xmn_percent: float (e.g 0.2)
   @param xmn_max: integer (e.g 512)
   """
-  heapsize = int(re.search('\d+', str(heapsize_str)).group(0))
-  heapsize_unit = re.search('\D+', str(heapsize_str)).group(0)
   xmn_val = int(math.floor(heapsize*xmn_percent))
   xmn_val -= xmn_val % 8
   
   result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
-  return str(result_xmn_val) + heapsize_unit
+  return result_xmn_val

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -32,15 +32,6 @@ else:
 config = Script.get_config()
 exec_tmp_dir = Script.get_tmp_dir()
 
-def get_combined_memory_mb(value1, value2):
-  try:
-    part1 = int(value1.strip()[:-1]) if value1.lower().strip()[-1:] == 'm' else int(value1)
-    part2 = int(value2.strip()[:-1]) if value2.lower().strip()[-1:] == 'm' else int(value2)
-    return str(part1 + part2) + 'm'
-  except:
-    return None
-pass
-
 #AMBARI_METRICS data
 ams_pid_dir = status_params.ams_collector_pid_dir
 
@@ -107,7 +98,7 @@ hbase_master_maxperm_size = config['configurations']['ams-hbase-env']['hbase_mas
 
 # Choose heap size for embedded mode as sum of master + regionserver
 if not is_hbase_distributed:
-  hbase_heapsize = get_combined_memory_mb(master_heapsize, regionserver_heapsize)
+  hbase_heapsize = master_heapsize + regionserver_heapsize
   if hbase_heapsize is None:
     hbase_heapsize = master_heapsize
 else:

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/split_points.py

@@ -44,14 +44,14 @@ def to_number(s):
   except ValueError:
     return None
 
-def format_Xmx_size_to_bytes(value):
+def format_Xmx_size_to_bytes(value, default='b'):
   strvalue = str(value).lower()
   if len(strvalue) == 0:
     return 0
   modifier = strvalue[-1]
 
   if modifier == ' ' or modifier in "0123456789":
-    modifier = 'b'
+    modifier = default
 
   m = {
     modifier == 'b': b_bytes,
@@ -88,10 +88,10 @@ class FindSplitPointsForAMSRegions():
 
   def initialize_region_counts(self):
     try:
-      xmx_master_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_master_heapsize'])
+      xmx_master_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_master_heapsize'], 'm')
       xmx_region_bytes = 0
       if "hbase_regionserver_heapsize" in self.ams_hbase_env:
-        xmx_region_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_regionserver_heapsize'])
+        xmx_region_bytes = format_Xmx_size_to_bytes(self.ams_hbase_env['hbase_regionserver_heapsize'], 'm')
       xmx_bytes = xmx_master_bytes + xmx_region_bytes
       if self.mode == 'distributed':
         xmx_bytes = xmx_region_bytes

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/functions.py

@@ -38,3 +38,17 @@ def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
   
   result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
   return str(result_xmn_val) + heapsize_unit
+
+def ensure_unit_for_memory(memory_size):
+  memory_size_values = re.findall('\d+', str(memory_size))
+  memory_size_unit = re.findall('\D+', str(memory_size))
+
+  if len(memory_size_values) > 0:
+    unit = 'm'
+    if len(memory_size_unit) > 0:
+      unit = memory_size_unit[0]
+    if unit not in ['b', 'k', 'm', 'g', 't', 'p']:
+      raise Exception("Memory size unit error. %s - wrong unit" % unit)
+    return "%s%s" % (memory_size_values[0], unit)
+  else:
+    raise Exception('Memory size can not be calculated')

ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/params_linux.py

@@ -20,7 +20,7 @@ limitations under the License.
 import status_params
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
 
-from functions import calc_xmn_from_xms
+from functions import calc_xmn_from_xms, ensure_unit_for_memory
 
 from ambari_commons.constants import AMBARI_SUDO_BINARY
 
@@ -101,9 +101,9 @@ java64_home = config['hostLevelParams']['java_home']
 java_version = int(config['hostLevelParams']['java_version'])
 
 log_dir = config['configurations']['hbase-env']['hbase_log_dir']
-master_heapsize = config['configurations']['hbase-env']['hbase_master_heapsize']
+master_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_master_heapsize'])
 
-regionserver_heapsize = config['configurations']['hbase-env']['hbase_regionserver_heapsize']
+regionserver_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_regionserver_heapsize'])
 regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max']
 regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio']
 regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max)

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/stack_advisor.py

@@ -351,7 +351,7 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
 
     collector_heapsize, hbase_heapsize, total_sinks_count = self.getAmsMemoryRecommendation(services, hosts)
 
-    putAmsEnvProperty("metrics_collector_heapsize", str(collector_heapsize) + "m")
+    putAmsEnvProperty("metrics_collector_heapsize", collector_heapsize)
 
     # blockCache = 0.3, memstore = 0.35, phoenix-server = 0.15, phoenix-client = 0.25
     putAmsHbaseSiteProperty("hfile.block.cache.size", 0.3)
@@ -373,26 +373,26 @@ class HDP206StackAdvisor(DefaultStackAdvisor):
         putAmsHbaseSiteProperty("hbase.regionserver.global.memstore.lowerLimit", 0.25)
         putAmsHbaseSiteProperty("phoenix.query.maxGlobalMemoryPercentage", 20)
         putTimelineServiceProperty("phoenix.query.maxGlobalMemoryPercentage", 30)
-        hbase_xmn_size = '512m'
+        hbase_xmn_size = '512'
       elif total_sinks_count >= 500:
         putAmsHbaseSiteProperty("hbase.regionserver.handler.count", 60)
         putAmsHbaseSiteProperty("hbase.regionserver.hlog.blocksize", 134217728)
         putAmsHbaseSiteProperty("hbase.regionserver.maxlogs", 64)
         putAmsHbaseSiteProperty("hbase.hregion.memstore.flush.size", 268435456)
-        hbase_xmn_size = '512m'
+        hbase_xmn_size = '512'
       elif total_sinks_count >= 250:
-        hbase_xmn_size = '256m'
+        hbase_xmn_size = '256'
       else:
-        hbase_xmn_size = '128m'
+        hbase_xmn_size = '128'
       pass
 
     # Embedded mode heap size : master + regionserver
     if rootDir.startswith("hdfs://"):
-      putHbaseEnvProperty("hbase_master_heapsize", "512m")
-      putHbaseEnvProperty("hbase_regionserver_heapsize", str(hbase_heapsize) + "m")
+      putHbaseEnvProperty("hbase_master_heapsize", "512")
+      putHbaseEnvProperty("hbase_regionserver_heapsize", hbase_heapsize)
       putHbaseEnvProperty("regionserver_xmn_size", hbase_xmn_size)
     else:
-      putHbaseEnvProperty("hbase_master_heapsize", str(hbase_heapsize) + "m")
+      putHbaseEnvProperty("hbase_master_heapsize", hbase_heapsize)
       putHbaseEnvProperty("hbase_master_xmn_size", hbase_xmn_size)
 
     # If no local DN in distributed mode
@@ -1217,10 +1217,10 @@ def getHeapsizeProperties():
                                  "default": "1024m"}],
            "METRICS_COLLECTOR": [{"config-name": "ams-hbase-env",
                                    "property": "hbase_master_heapsize",
-                                   "default": "1024m"},
+                                   "default": "1024"},
                                  {"config-name": "ams-env",
                                    "property": "metrics_collector_heapsize",
-                                   "default": "512m"}],
+                                   "default": "512"}],
            }
 
 def getMemorySizeRequired(components, configurations):

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java

@@ -18,6 +18,7 @@
 
 package org.apache.ambari.server.upgrade;
 
+import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.createMockBuilder;
 import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.createStrictMock;
@@ -27,6 +28,7 @@ import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.reset;
 import static org.easymock.EasyMock.verify;
 
+import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.HashMap;
 import java.util.Map;
@@ -46,6 +48,7 @@ import org.apache.ambari.server.state.Clusters;
 import org.apache.ambari.server.state.Config;
 import org.apache.ambari.server.state.ConfigHelper;
 import org.apache.ambari.server.state.stack.OsFamily;
+import org.easymock.EasyMock;
 import org.easymock.EasyMockSupport;
 import org.junit.After;
 import org.junit.Assert;
@@ -93,13 +96,17 @@ public class UpgradeCatalog213Test {
   @Test
   public void testExecuteDMLUpdates() throws Exception {
     Method addMissingConfigs = UpgradeCatalog213.class.getDeclaredMethod("addMissingConfigs");
+    Method updateAMSConfigs = UpgradeCatalog213.class.getDeclaredMethod("updateAMSConfigs");
 
     UpgradeCatalog213 upgradeCatalog213 = createMockBuilder(UpgradeCatalog213.class)
         .addMockedMethod(addMissingConfigs)
+        .addMockedMethod(updateAMSConfigs)
         .createMock();
 
     upgradeCatalog213.addMissingConfigs();
     expectLastCall().once();
+    upgradeCatalog213.updateAMSConfigs();
+    expectLastCall().once();
 
     replay(upgradeCatalog213);
 
@@ -154,6 +161,103 @@ public class UpgradeCatalog213Test {
     easyMockSupport.verifyAll();
   }
 
+  @Test
+  public void testUpdateAmsHbaseEnvContent() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+    Method updateAmsHbaseEnvContent = UpgradeCatalog213.class.getDeclaredMethod("updateAmsHbaseEnvContent", String.class);
+    UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
+    String oldContent = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}\n" +
+            "\n" +
+            "# The maximum amount of heap to use, in MB. Default is 1000.\n" +
+            "export HBASE_HEAPSIZE={{hbase_heapsize}}\n" +
+            "\n" +
+            "{% if java_version < 8 %}\n" +
+            "export HBASE_MASTER_OPTS=\" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}} -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
+            "export HBASE_REGIONSERVER_OPTS=\"-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n" +
+            "{% else %}\n" +
+            "export HBASE_MASTER_OPTS=\" -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
+            "export HBASE_REGIONSERVER_OPTS=\" -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n" +
+            "{% endif %}\n";
+    String expectedContent = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}\n" +
+            "\n" +
+            "# The maximum amount of heap to use, in MB. Default is 1000.\n" +
+            "export HBASE_HEAPSIZE={{hbase_heapsize}}m\n" +
+            "\n" +
+            "{% if java_version < 8 %}\n" +
+            "export HBASE_MASTER_OPTS=\" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}}m -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
+            "export HBASE_REGIONSERVER_OPTS=\"-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m\"\n" +
+            "{% else %}\n" +
+            "export HBASE_MASTER_OPTS=\" -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
+            "export HBASE_REGIONSERVER_OPTS=\" -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m\"\n" +
+            "{% endif %}\n";
+    String result = (String) updateAmsHbaseEnvContent.invoke(upgradeCatalog213, oldContent);
+    Assert.assertEquals(expectedContent, result);
+  }
+
+  @Test
+  public void testUpdateAmsEnvContent() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
+    Method updateAmsEnvContent = UpgradeCatalog213.class.getDeclaredMethod("updateAmsEnvContent", String.class);
+    UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
+    String oldContent = "# AMS Collector heapsize\n" +
+            "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n";
+    String expectedContent = "# AMS Collector heapsize\n" +
+            "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}m\n";
+    String result = (String) updateAmsEnvContent.invoke(upgradeCatalog213, oldContent);
+    Assert.assertEquals(expectedContent, result);
+  }
+
+  @Test
+  public void testUpdateAmsConfigs() throws Exception {
+    EasyMockSupport easyMockSupport = new EasyMockSupport();
+    final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
+    final ConfigHelper mockConfigHelper = easyMockSupport.createMock(ConfigHelper.class);
+
+    final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
+    final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
+    final Map<String, String> propertiesAmsEnv = new HashMap<String, String>() {
+      {
+        put("metrics_collector_heapsize", "512m");
+      }
+    };
+
+    final Map<String, String> propertiesAmsHbaseEnv = new HashMap<String, String>() {
+      {
+        put("hbase_regionserver_heapsize", "512m");
+        put("regionserver_xmn_size", "512m");
+        put("hbase_master_xmn_size", "512m");
+        put("hbase_master_maxperm_size", "512");
+      }
+    };
+
+    final Config mockAmsEnv = easyMockSupport.createNiceMock(Config.class);
+    final Config mockAmsHbaseEnv = easyMockSupport.createNiceMock(Config.class);
+
+    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
+      @Override
+      protected void configure() {
+        bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
+        bind(ConfigHelper.class).toInstance(mockConfigHelper);
+        bind(Clusters.class).toInstance(mockClusters);
+
+        bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
+        bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
+      }
+    });
+
+    expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once();
+    expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
+      put("normal", mockClusterExpected);
+    }}).once();
+
+    expect(mockClusterExpected.getDesiredConfigByType("ams-env")).andReturn(mockAmsEnv).atLeastOnce();
+    expect(mockClusterExpected.getDesiredConfigByType("ams-hbase-env")).andReturn(mockAmsHbaseEnv).atLeastOnce();
+    expect(mockAmsEnv.getProperties()).andReturn(propertiesAmsEnv).atLeastOnce();
+    expect(mockAmsHbaseEnv.getProperties()).andReturn(propertiesAmsHbaseEnv).atLeastOnce();
+
+    easyMockSupport.replayAll();
+    mockInjector.getInstance(UpgradeCatalog213.class).updateAMSConfigs();
+    easyMockSupport.verifyAll();
+  }
+
   /**
    * @param dbAccessor
    * @return

ambari-server/src/test/python/stacks/2.2/common/test_stack_advisor.py

@@ -1983,13 +1983,13 @@ class TestHDP22StackAdvisor(TestCase):
     expected = {
       "ams-hbase-env": {
         "properties": {
-          "hbase_master_xmn_size": "128m",
-          "hbase_master_heapsize": "512m"
+          "hbase_master_xmn_size": "128",
+          "hbase_master_heapsize": "512"
         }
       },
       "ams-env": {
         "properties": {
-          "metrics_collector_heapsize": "512m",
+          "metrics_collector_heapsize": "512",
         }
       },
       "ams-hbase-site": {
@@ -2042,8 +2042,8 @@ class TestHDP22StackAdvisor(TestCase):
         ]
       }
     ]
-    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1408m'
-    expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512m'
+    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '1408'
+    expected["ams-env"]['properties']['metrics_collector_heapsize'] = '512'
 
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
@@ -2109,9 +2109,9 @@ class TestHDP22StackAdvisor(TestCase):
       }
 
     ]
-    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2432m'
-    expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '256m'
-    expected["ams-env"]['properties']['metrics_collector_heapsize'] = '640m'
+    expected["ams-hbase-env"]['properties']['hbase_master_heapsize'] = '2432'
+    expected["ams-hbase-env"]['properties']['hbase_master_xmn_size'] = '256'
+    expected["ams-env"]['properties']['metrics_collector_heapsize'] = '640'
 
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
@@ -2131,29 +2131,29 @@ class TestHDP22StackAdvisor(TestCase):
     }
 
     # Embedded mode, 512m master heapsize, no splitpoints recommended
-    services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512m'
+    services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512'
     services["configurations"]['ams-hbase-site']['properties']['hbase.regionserver.global.memstore.lowerLimit'] = '0.3'
     services["configurations"]['ams-hbase-site']['properties']['hbase.hregion.memstore.flush.size'] = '134217728'
 
     expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = ' '
     expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = ' '
-    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512m'
+    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512'
 
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
     # Embedded mode, 4096m master heapsize, some splitpoints recommended
-    services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096m'
+    services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096'
     expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = \
       'jvm.JvmMetrics.MemHeapCommittedM,regionserver.Server.Increment_median'
     expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = ' '
-    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096m'
+    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '4096'
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
     # Embedded mode, 8192m master heapsize, more splitpoints recommended
-    services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192m'
-    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192m'
+    services["configurations"]['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192'
+    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '8192'
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')), 10)
     self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')), 2)
@@ -2168,22 +2168,22 @@ class TestHDP22StackAdvisor(TestCase):
     ]
     services["configurations"]['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase'
     expected['ams-hbase-site']['properties']['hbase.rootdir'] = 'hdfs://host1/amshbase'
-    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512m'
+    expected['ams-hbase-env']['properties']['hbase_master_heapsize'] = '512'
     # services["configurations"]['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true'
     expected['ams-hbase-site']['properties']['dfs.client.read.shortcircuit'] = 'true'
 
     # Distributed mode, low memory, no splitpoints recommended
-    services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512m'
+    services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512'
     expected['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'] = ' '
     expected['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'] = ' '
-    expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512m'
-    expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '256m'
+    expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '512'
+    expected['ams-hbase-env']['properties']['regionserver_xmn_size'] = '256'
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(configurations, expected)
 
     # Distributed mode, more memory, more splitpoints recommended
-    services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192m'
-    expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192m'
+    services["configurations"]['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192'
+    expected['ams-hbase-env']['properties']['hbase_regionserver_heapsize'] = '8192'
     self.stackAdvisor.recommendAmsConfigurations(configurations, clusterData, services, hosts)
     self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.host.aggregate.splitpoints'].split(',')), 10)
     self.assertEquals(len(configurations['ams-site']['properties']['timeline.metrics.cluster.aggregate.splitpoints'].split(',')), 2)

ambari-web/app/controllers/wizard/step8_controller.js

@@ -1585,7 +1585,7 @@ App.WizardStep8Controller = Em.Controller.extend(App.AddSecurityConfigs, App.wiz
     var configs = this.get('configs').filterProperty('filename', site + '.xml');
     var attributes = App.router.get('mainServiceInfoConfigsController').getConfigAttributes(configs);
     configs.forEach(function (_configProperty) {
-        var heapsizeExceptions = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize', 'nfsgateway_heapsize', 'accumulo_master_heapsize', 'accumulo_tserver_heapsize', 'accumulo_monitor_heapsize', 'accumulo_gc_heapsize', 'accumulo_other_heapsize'];
+        var heapsizeExceptions = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize', 'nfsgateway_heapsize', 'accumulo_master_heapsize', 'accumulo_tserver_heapsize', 'accumulo_monitor_heapsize', 'accumulo_gc_heapsize', 'accumulo_other_heapsize', 'hbase_master_heapsize', 'hbase_regionserver_heapsize'];
         // do not pass any globals whose name ends with _host or _hosts
         if (_configProperty.isRequiredByAgent !== false) {
           // append "m" to JVM memory options except for heapsizeExtensions

ambari-web/app/mixins/common/configs/configs_saver.js

@@ -50,7 +50,7 @@ App.ConfigsSaverMixin = Em.Mixin.create({
    * List of heapsize properties not to be parsed
    * @type {string[]}
    */
-  heapsizeException: ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize', 'nfsgateway_heapsize', 'accumulo_master_heapsize', 'accumulo_tserver_heapsize', 'accumulo_monitor_heapsize', 'accumulo_gc_heapsize', 'accumulo_other_heapsize'],
+  heapsizeException: ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize', 'nfsgateway_heapsize', 'accumulo_master_heapsize', 'accumulo_tserver_heapsize', 'accumulo_monitor_heapsize', 'accumulo_gc_heapsize', 'accumulo_other_heapsize', 'hbase_master_heapsize', 'hbase_regionserver_heapsize'],
 
   /**
    * Regular expression for heapsize properties detection