AMBARI-5485. Ambari should Manage JobHistory Heap Size (aonishuk)

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/params.py

@@ -105,6 +105,9 @@ if rca_enabled == True:
 else:
   rca_prefix = rca_disabled_prefix
 
+#mapred-env.sh
+jobhistory_heapsize = config['configurations']['global']['jobhistory_heapsize']
+
 #hadoop-env.sh
 java_home = config['hostLevelParams']['java_home']
 

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/scripts/shared_initialization.py

@@ -60,7 +60,7 @@ def setup_hadoop():
   else:
     tc_owner = params.hdfs_user
 
-  for file in ['hadoop-env.sh', 'commons-logging.properties']:
+  for file in ['hadoop-env.sh', 'commons-logging.properties', 'mapred-env.sh']:
     File(os.path.join(params.hadoop_conf_dir, file),
          owner=tc_owner,
          content=Template(file + ".j2")

ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-START/templates/mapred-env.sh.j2

@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# export JAVA_HOME=/home/y/libexec/jdk1.6.0/
+
+export HADOOP_JOB_HISTORYSERVER_HEAPSIZE={{jobhistory_heapsize}}
+
+export HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA
+
+#export HADOOP_JOB_HISTORYSERVER_OPTS=
+#export HADOOP_MAPRED_LOG_DIR="" # Where log files are stored.  $HADOOP_MAPRED_HOME/logs by default.
+#export HADOOP_JHS_LOGGER=INFO,RFA # Hadoop JobSummary logger.
+#export HADOOP_MAPRED_PID_DIR= # The pid files are stored. /tmp by default.
+#export HADOOP_MAPRED_IDENT_STRING= #A string representing this instance of hadoop. $USER by default
+#export HADOOP_MAPRED_NICENESS= #The scheduling priority for daemons. Defaults to 0.

ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/configuration-mapred/global.xml

@@ -36,5 +36,9 @@
     <value>mapred</value>
     <description>Mapreduce User</description>
   </property>
-
+  <property>
+    <name>jobhistory_heapsize</name>
+    <value>900</value>
+    <description>Value for JobHistoryServer heap_size variable in hadoop-env.sh</description>
+  </property>
 </configuration>

ambari-server/src/test/python/stacks/2.0.6/hooks/before-START/test_before_start.py

@@ -63,6 +63,10 @@ class TestHookBeforeStart(RMFTestCase):
                               content = Template('commons-logging.properties.j2'),
                               owner = 'hdfs',
                               )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'hdfs',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
                               content = Template('health_check-v2.j2'),
                               owner = 'hdfs',
@@ -145,6 +149,10 @@ class TestHookBeforeStart(RMFTestCase):
                               content = Template('commons-logging.properties.j2'),
                               owner = 'root',
                               )
+    self.assertResourceCalled('File', '/etc/hadoop/conf/mapred-env.sh',
+                              content = Template('mapred-env.sh.j2'),
+                              owner = 'root',
+                              )
     self.assertResourceCalled('File', '/etc/hadoop/conf/health_check',
                               content = Template('health_check-v2.j2'),
                               owner = 'root',

ambari-web/app/controllers/main/service/info/configs.js

@@ -1444,7 +1444,7 @@ App.MainServiceInfoConfigsController = Em.Controller.extend({
    * @return {Object}
    */
   createGlobalSiteObj: function (tagName, globalConfigs) {
-    var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize'];
+    var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize'];
     var globalSiteProperties = {};
     globalConfigs.forEach(function (_globalSiteObj) {
       // do not pass any globalConfigs whose name ends with _host or _hosts

ambari-web/app/controllers/wizard/step8_controller.js

@@ -1510,7 +1510,7 @@ App.WizardStep8Controller = Em.Controller.extend({
     }
 
     globalSiteObj.forEach(function (_globalSiteObj) {
-      var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize'];
+      var heapsizeException = ['hadoop_heapsize', 'yarn_heapsize', 'nodemanager_heapsize', 'resourcemanager_heapsize', 'apptimelineserver_heapsize', 'jobhistory_heapsize'];
       // do not pass any globals whose name ends with _host or _hosts
       if (_globalSiteObj.isRequiredByAgent !== false) {
         // append "m" to JVM memory options except for hadoop_heapsize

ambari-web/app/data/HDP2/global_properties.js

@@ -274,6 +274,20 @@ module.exports =
       "category": "HistoryServer",
       "index": 0
     },
+    {
+      "id": "puppet var",
+      "name": "jobhistory_heapsize",
+      "displayName": "History Server heap size",
+      "description": "History Server heap size",
+      "defaultValue": "900",
+      "unit": "MB",
+      "isOverridable": true,
+      "displayType": "int",
+      "isVisible": true,
+      "serviceName": "MAPREDUCE2",
+      "category": "HistoryServer",
+      "index": 1
+    },
     {
       "id": "puppet var",
       "name": "mapred_log_dir_prefix",