AMBARI-8017. Upgrades : Not enough heap size for NameNode At EC2 cluster (aonishuk)

ambari-common/src/main/python/resource_management/libraries/functions/__init__.py

@@ -30,3 +30,4 @@ from resource_management.libraries.functions.substitute_vars import *
 from resource_management.libraries.functions.get_port_from_url import *
 from resource_management.libraries.functions.hive_check import *
 from resource_management.libraries.functions.version import *
+from resource_management.libraries.functions.format_jvm_option import *

ambari-common/src/main/python/resource_management/libraries/functions/format_jvm_option.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+__all__ = ["format_jvm_option"]
+from resource_management.libraries.script import Script
+from resource_management.libraries.script.config_dictionary import UnknownConfiguration
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import *
+
+def format_jvm_option(name, default_value):
+  curr_dict = default(name, default_value)
+  if not "m" in curr_dict:
+    curr_dict = curr_dict + m
+  
+  return curr_dict

ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/params.py

@@ -48,8 +48,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = "200m"
 jtnode_opt_maxnewsize = "200m"

ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/params.py

@@ -119,8 +119,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = "200m"
 jtnode_opt_maxnewsize = "200m"

ambari-server/src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/params.py

@@ -201,8 +201,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = "200m"
 jtnode_opt_maxnewsize = "200m"

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/params.py

@@ -44,8 +44,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
 

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-ANY/scripts/params.py

@@ -56,8 +56,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = default("/configurations/mapred-env/jtnode_opt_newsize","200m")
 jtnode_opt_maxnewsize = default("/configurations/mapred-env/jtnode_opt_maxnewsize","200m")

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-INSTALL/scripts/params.py

@@ -53,8 +53,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
 mapred_pid_dir_prefix = default("/configurations/hadoop-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")

ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py

@@ -115,8 +115,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
 mapred_pid_dir_prefix = "/var/run/hadoop-mapreduce"

ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py

@@ -176,8 +176,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = default("/configurations/mapred-env/jtnode_opt_newsize","200m")
 jtnode_opt_maxnewsize = default("/configurations/mapred-env/jtnode_opt_maxnewsize","200m")

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/params.py

@@ -55,8 +55,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = "200m"
 jtnode_opt_maxnewsize = "200m"

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-ANY/scripts/params.py

@@ -66,8 +66,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = "200m"
 jtnode_opt_maxnewsize = "200m"

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py

@@ -133,8 +133,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = "200m"
 jtnode_opt_maxnewsize = "200m"

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py

@@ -238,8 +238,8 @@ hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
 namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
 namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
 namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
-namenode_opt_permsize = config['configurations']['hadoop-env']['namenode_opt_permsize']
-namenode_opt_maxpermsize = config['configurations']['hadoop-env']['namenode_opt_maxpermsize']
+namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m")
+namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m")
 
 jtnode_opt_newsize = "200m"
 jtnode_opt_maxnewsize = "200m"