AMBARI-14120 : AMS Collector and HBase heap memory config properties are incompatible between 2.1.2 and 2.1.3 when deployed through the same blueprint. (Aravindan Vijayan via swagle)

ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog213.java

@@ -1009,36 +1009,6 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
 
       if (clusterMap != null && !clusterMap.isEmpty()) {
         for (final Cluster cluster : clusterMap.values()) {
-          Config amsEnv = cluster.getDesiredConfigByType(AMS_ENV);
-          if (amsEnv != null) {
-            Map<String, String> amsEnvProperties = amsEnv.getProperties();
-
-            String metrics_collector_heapsize = amsEnvProperties.get("metrics_collector_heapsize");
-            String content = amsEnvProperties.get("content");
-            Map<String, String> newProperties = new HashMap<>();
-            newProperties.put("metrics_collector_heapsize", memoryToIntMb(metrics_collector_heapsize));
-            newProperties.put("content", updateAmsEnvContent(content));
-            updateConfigurationPropertiesForCluster(cluster, AMS_ENV, newProperties, true, true);
-          }
-          Config amsHbaseEnv = cluster.getDesiredConfigByType(AMS_HBASE_ENV);
-          if (amsHbaseEnv != null) {
-            Map<String, String> amsHbaseEnvProperties = amsHbaseEnv.getProperties();
-            String hbase_regionserver_heapsize = amsHbaseEnvProperties.get("hbase_regionserver_heapsize");
-            String regionserver_xmn_size = amsHbaseEnvProperties.get("regionserver_xmn_size");
-            String hbase_master_xmn_size = amsHbaseEnvProperties.get("hbase_master_xmn_size");
-            String hbase_master_maxperm_size = amsHbaseEnvProperties.get("hbase_master_maxperm_size");
-            String hbase_master_heapsize = amsHbaseEnvProperties.get("hbase_master_heapsize");
-            String content = amsHbaseEnvProperties.get("content");
-
-            Map<String, String> newProperties = new HashMap<>();
-            newProperties.put("hbase_regionserver_heapsize", memoryToIntMb(hbase_regionserver_heapsize));
-            newProperties.put("regionserver_xmn_size", memoryToIntMb(regionserver_xmn_size));
-            newProperties.put("hbase_master_xmn_size", memoryToIntMb(hbase_master_xmn_size));
-            newProperties.put("hbase_master_maxperm_size", memoryToIntMb(hbase_master_maxperm_size));
-            newProperties.put("hbase_master_heapsize", memoryToIntMb(hbase_master_heapsize));
-            newProperties.put("content", updateAmsHbaseEnvContent(content));
-            updateConfigurationPropertiesForCluster(cluster, AMS_HBASE_ENV, newProperties, true, true);
-          }
           Config amsSite = cluster.getDesiredConfigByType(AMS_SITE);
           if (amsSite != null) {
             Map<String, String> newProperties = new HashMap<>();
@@ -1155,44 +1125,6 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
     }
   }
 
-  protected String updateAmsEnvContent(String oldContent) {
-    if (oldContent == null) {
-      return null;
-    }
-    String regSearch = "export\\s*AMS_COLLECTOR_HEAPSIZE\\s*=\\s*\\{\\{metrics_collector_heapsize\\}\\}";
-    String replacement = "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}m";
-    return oldContent.replaceAll(regSearch, replacement);
-  }
-
-  protected String updateAmsHbaseEnvContent(String content) {
-    if (content == null) {
-      return null;
-    }
-
-    String regSearch = "\\{\\{hbase_heapsize\\}\\}";
-    String replacement = "{{hbase_heapsize}}m";
-    content = content.replaceAll(regSearch, replacement);
-    regSearch = "\\{\\{hbase_master_maxperm_size\\}\\}";
-    replacement = "{{hbase_master_maxperm_size}}m";
-    content = content.replaceAll(regSearch, replacement);
-    regSearch = "\\{\\{hbase_master_xmn_size\\}\\}";
-    replacement = "{{hbase_master_xmn_size}}m";
-    content = content.replaceAll(regSearch, replacement);
-    regSearch = "\\{\\{regionserver_xmn_size\\}\\}";
-    replacement = "{{regionserver_xmn_size}}m";
-    content = content.replaceAll(regSearch, replacement);
-    regSearch = "\\{\\{regionserver_heapsize\\}\\}";
-    replacement = "{{regionserver_heapsize}}m";
-    content = content.replaceAll(regSearch, replacement);
-    regSearch = "export HBASE_HEAPSIZE=";
-    replacement = "#export HBASE_HEAPSIZE=";
-    content = content.replaceAll(regSearch, replacement);
-    content += "\n" +
-      "# The maximum amount of heap to use for hbase shell.\n" +
-      "export HBASE_SHELL_OPTS=\"-Xmx256m\"\n";
-    return content;
-  }
-
   protected String updateHiveEnvContent(String hiveEnvContent) {
     if(hiveEnvContent == null) {
       return null;
@@ -1252,33 +1184,4 @@ public class UpgradeCatalog213 extends AbstractUpgradeCatalog {
       } // else -- no special client-configuration is necessary.
     }
   }
-
-  private String memoryToIntMb(String memorySize) {
-    if (memorySize == null) {
-      return "0";
-    }
-    Integer value = 0;
-    try {
-      value = Integer.parseInt(memorySize.replaceAll("\\D+", ""));
-    } catch (NumberFormatException ex) {
-      LOG.error(ex.getMessage());
-    }
-    char unit = memorySize.toUpperCase().charAt(memorySize.length() - 1);
-    // Recalculate memory size to Mb
-    switch (unit) {
-      case 'K':
-        value /= 1024;
-        break;
-      case 'B':
-        value /= (1024*1024);
-        break;
-      case 'G':
-        value *= 1024;
-        break;
-      case 'T':
-        value *= 1024*1024;
-        break;
-    }
-    return value.toString();
-  }
 }

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-env.xml

@@ -83,7 +83,7 @@ export AMS_MONITOR_PID_DIR={{ams_monitor_pid_dir}}
 export AMS_HBASE_PID_DIR={{hbase_pid_dir}}
 
 # AMS Collector heapsize
-export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}m
+export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}
 
 # AMS Collector options
 export AMS_COLLECTOR_OPTS="-Djava.library.path=/usr/lib/ams-hbase/lib/hadoop-native"

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-hbase-env.xml

@@ -178,11 +178,11 @@ export SERVER_GC_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -X
 # export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
 
 {% if java_version < 8 %}
-export HBASE_MASTER_OPTS=" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}}m -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
-export HBASE_REGIONSERVER_OPTS="-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m"
+export HBASE_MASTER_OPTS=" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}} -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
+export HBASE_REGIONSERVER_OPTS="-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
 {% else %}
-export HBASE_MASTER_OPTS=" -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
-export HBASE_REGIONSERVER_OPTS=" -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m"
+export HBASE_MASTER_OPTS=" -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly"
+export HBASE_REGIONSERVER_OPTS=" -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}"
 {% endif %}
 
 

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/functions.py

@@ -25,21 +25,27 @@ import datetime
 
 from resource_management.core.shell import checked_call
 
-def calc_xmn_from_xms(heapsize, xmn_percent, xmn_max):
+def calc_xmn_from_xms(heapsize_str, xmn_percent, xmn_max):
   """
-  @param heapsize: int (e.g 1000)
+  @param heapsize: str (e.g 1000m)
   @param xmn_percent: float (e.g 0.2)
   @param xmn_max: integer (e.g 512)
   """
+  heapsize = int(re.search('\d+', str(heapsize_str)).group(0))
+  heapsize_unit = re.search('\D+', str(heapsize_str)).group(0)
+
   xmn_val = int(math.floor(heapsize*xmn_percent))
   xmn_val -= xmn_val % 8
   
   result_xmn_val = xmn_max if xmn_val > xmn_max else xmn_val
-  return result_xmn_val
+  return str(result_xmn_val) + heapsize_unit
 
 def trim_heap_property(property, m_suffix = "m"):
-
   if property and property.endswith(m_suffix):
     property = property[:-1]
-
   return property
+
+def check_append_heap_property(property, m_suffix = "m"):
+  if property and not property.endswith(m_suffix):
+    property += m_suffix
+  return property

ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py

@@ -19,6 +19,7 @@ limitations under the License.
 """
 
 from functions import calc_xmn_from_xms
+from functions import check_append_heap_property
 from functions import trim_heap_property
 
 from resource_management import *
@@ -34,6 +35,15 @@ else:
 config = Script.get_config()
 exec_tmp_dir = Script.get_tmp_dir()
 
+def get_combined_memory_mb(value1, value2):
+  try:
+    part1 = int(value1.strip()[:-1]) if value1.lower().strip()[-1:] == 'm' else int(value1)
+    part2 = int(value2.strip()[:-1]) if value2.lower().strip()[-1:] == 'm' else int(value2)
+    return str(part1 + part2) + 'm'
+  except:
+    return None
+pass
+
 #AMBARI_METRICS data
 ams_pid_dir = status_params.ams_collector_pid_dir
 
@@ -99,9 +109,9 @@ master_heapsize = config['configurations']['ams-hbase-env']['hbase_master_heapsi
 regionserver_heapsize = config['configurations']['ams-hbase-env']['hbase_regionserver_heapsize']
 
 # Check if hbase java options already have appended "m". If Yes, remove the trailing m.
-metrics_collector_heapsize = int(trim_heap_property(str(metrics_collector_heapsize), "m"))
-master_heapsize = int(trim_heap_property(str(master_heapsize), "m"))
-regionserver_heapsize = int(trim_heap_property(str(regionserver_heapsize), "m"))
+metrics_collector_heapsize = check_append_heap_property(str(metrics_collector_heapsize), "m")
+master_heapsize = check_append_heap_property(str(master_heapsize), "m")
+regionserver_heapsize = check_append_heap_property(str(regionserver_heapsize), "m")
 
 regionserver_xmn_max = default('/configurations/ams-hbase-env/hbase_regionserver_xmn_max', None)
 if regionserver_xmn_max:
@@ -116,13 +126,13 @@ hbase_master_xmn_size = config['configurations']['ams-hbase-env']['hbase_master_
 hbase_master_maxperm_size = config['configurations']['ams-hbase-env']['hbase_master_maxperm_size']
 
 # Check if hbase java options already have appended "m". If Yes, remove the trailing m.
-hbase_master_maxperm_size = int(trim_heap_property(str(hbase_master_maxperm_size), "m"))
-hbase_master_xmn_size = int(trim_heap_property(str(hbase_master_xmn_size), "m"))
-regionserver_xmn_size = int(trim_heap_property(str(regionserver_xmn_size), "m"))
+hbase_master_maxperm_size = check_append_heap_property(str(hbase_master_maxperm_size), "m")
+hbase_master_xmn_size = check_append_heap_property(str(hbase_master_xmn_size), "m")
+regionserver_xmn_size = check_append_heap_property(str(regionserver_xmn_size), "m")
 
 # Choose heap size for embedded mode as sum of master + regionserver
 if not is_hbase_distributed:
-  hbase_heapsize = master_heapsize + regionserver_heapsize
+  hbase_heapsize = get_combined_memory_mb(master_heapsize, regionserver_heapsize)
   if hbase_heapsize is None:
     hbase_heapsize = master_heapsize
 else:

ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog213Test.java

@@ -702,106 +702,6 @@ public class UpgradeCatalog213Test {
     easyMockSupport.verifyAll();
   }
 
-  @Test
-  public void testUpdateAmsHbaseEnvContent() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
-    Method updateAmsHbaseEnvContent = UpgradeCatalog213.class.getDeclaredMethod("updateAmsHbaseEnvContent", String.class);
-    UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
-    String oldContent = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}\n" +
-        "\n" +
-        "# The maximum amount of heap to use, in MB. Default is 1000.\n" +
-        "export HBASE_HEAPSIZE={{hbase_heapsize}}\n" +
-        "\n" +
-        "{% if java_version < 8 %}\n" +
-        "export HBASE_MASTER_OPTS=\" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}} -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
-        "export HBASE_REGIONSERVER_OPTS=\"-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n" +
-        "{% else %}\n" +
-        "export HBASE_MASTER_OPTS=\" -Xms{{hbase_heapsize}} -Xmx{{hbase_heapsize}} -Xmn{{hbase_master_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
-        "export HBASE_REGIONSERVER_OPTS=\" -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n" +
-        "{% endif %}\n";
-    String expectedContent = "export HBASE_CLASSPATH=${HBASE_CLASSPATH}\n" +
-        "\n" +
-        "# The maximum amount of heap to use, in MB. Default is 1000.\n" +
-        "#export HBASE_HEAPSIZE={{hbase_heapsize}}m\n" +
-        "\n" +
-        "{% if java_version < 8 %}\n" +
-        "export HBASE_MASTER_OPTS=\" -XX:PermSize=64m -XX:MaxPermSize={{hbase_master_maxperm_size}}m -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
-        "export HBASE_REGIONSERVER_OPTS=\"-XX:MaxPermSize=128m -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m\"\n" +
-        "{% else %}\n" +
-        "export HBASE_MASTER_OPTS=\" -Xms{{hbase_heapsize}}m -Xmx{{hbase_heapsize}}m -Xmn{{hbase_master_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly\"\n" +
-        "export HBASE_REGIONSERVER_OPTS=\" -Xmn{{regionserver_xmn_size}}m -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -Xms{{regionserver_heapsize}}m -Xmx{{regionserver_heapsize}}m\"\n" +
-      "{% endif %}\n\n" +
-      "# The maximum amount of heap to use for hbase shell.\n" +
-      "export HBASE_SHELL_OPTS=\"-Xmx256m\"\n";
-    String result = (String) updateAmsHbaseEnvContent.invoke(upgradeCatalog213, oldContent);
-    Assert.assertEquals(expectedContent, result);
-  }
-
-  @Test
-  public void testUpdateAmsEnvContent() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
-    Method updateAmsEnvContent = UpgradeCatalog213.class.getDeclaredMethod("updateAmsEnvContent", String.class);
-    UpgradeCatalog213 upgradeCatalog213 = new UpgradeCatalog213(injector);
-    String oldContent = "# AMS Collector heapsize\n" +
-        "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}\n";
-    String expectedContent = "# AMS Collector heapsize\n" +
-        "export AMS_COLLECTOR_HEAPSIZE={{metrics_collector_heapsize}}m\n";
-    String result = (String) updateAmsEnvContent.invoke(upgradeCatalog213, oldContent);
-    Assert.assertEquals(expectedContent, result);
-  }
-
-  @Test
-  public void testUpdateAmsConfigs() throws Exception {
-    EasyMockSupport easyMockSupport = new EasyMockSupport();
-    final AmbariManagementController mockAmbariManagementController = easyMockSupport.createNiceMock(AmbariManagementController.class);
-    final ConfigHelper mockConfigHelper = easyMockSupport.createMock(ConfigHelper.class);
-
-    final Clusters mockClusters = easyMockSupport.createStrictMock(Clusters.class);
-    final Cluster mockClusterExpected = easyMockSupport.createNiceMock(Cluster.class);
-    final Map<String, String> propertiesAmsEnv = new HashMap<String, String>() {
-      {
-        put("metrics_collector_heapsize", "512m");
-      }
-    };
-
-    final Map<String, String> propertiesAmsHbaseEnv = new HashMap<String, String>() {
-      {
-        put("hbase_regionserver_heapsize", "512m");
-        put("regionserver_xmn_size", "512m");
-        put("hbase_master_xmn_size", "512m");
-        put("hbase_master_maxperm_size", "512");
-      }
-    };
-
-    final Config mockAmsEnv = easyMockSupport.createNiceMock(Config.class);
-    final Config mockAmsHbaseEnv = easyMockSupport.createNiceMock(Config.class);
-
-    final Injector mockInjector = Guice.createInjector(new AbstractModule() {
-      @Override
-      protected void configure() {
-        bind(AmbariManagementController.class).toInstance(mockAmbariManagementController);
-        bind(ConfigHelper.class).toInstance(mockConfigHelper);
-        bind(Clusters.class).toInstance(mockClusters);
-        bind(EntityManager.class).toInstance(entityManager);
-
-        bind(DBAccessor.class).toInstance(createNiceMock(DBAccessor.class));
-        bind(OsFamily.class).toInstance(createNiceMock(OsFamily.class));
-      }
-    });
-
-    expect(mockAmbariManagementController.getClusters()).andReturn(mockClusters).once();
-    expect(mockClusters.getClusters()).andReturn(new HashMap<String, Cluster>() {{
-      put("normal", mockClusterExpected);
-    }}).once();
-
-    expect(mockClusterExpected.getDesiredConfigByType("ams-env")).andReturn(mockAmsEnv).atLeastOnce();
-    expect(mockClusterExpected.getDesiredConfigByType("ams-hbase-env")).andReturn(mockAmsHbaseEnv).atLeastOnce();
-    expect(mockAmsEnv.getProperties()).andReturn(propertiesAmsEnv).atLeastOnce();
-    expect(mockAmsHbaseEnv.getProperties()).andReturn(propertiesAmsHbaseEnv).atLeastOnce();
-
-    easyMockSupport.replayAll();
-    mockInjector.getInstance(UpgradeCatalog213.class).updateAMSConfigs();
-    easyMockSupport.verifyAll();
-  }
-
   @Test
   public void testAmsSiteUpdateConfigs() throws Exception{