AMBARI-2047. Create ambari agent scripts for historyserver. (swagle)

git-svn-id: https://svn.apache.org/repos/asf/incubator/ambari/trunk@1477429 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -12,6 +12,8 @@ Trunk (unreleased changes):
 
  NEW FEATURES
 
+ AMBARI-2047. Create ambari agent scripts for historyserver. (swagle)
+
  AMBARI-2046. Create ambari agent scripts for Hadoop 2.0 installation, node 
  manager. (swagle)
 

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/namenode.pp

@@ -167,14 +167,37 @@ define hdp-hadoop::namenode::create_app_directories($service_state)
     }
 
     if $stack_version in ("2.0.1") {
-      if ($hdp::params::yarn_log_aggregation_enabled == "true") {
-        $yarn_user = $hdp::params::yarn_user
-        $yarn_nm_app_log_dir = $hdp::params::yarn_nm_app_log_dir
+      if ($hdp::params::nm_host != "") {
+        if ($hdp::params::yarn_log_aggregation_enabled == "true") {
+          $yarn_user = $hdp::params::yarn_user
+          $yarn_nm_app_log_dir = $hdp::params::yarn_nm_app_log_dir
 
-        hdp-hadoop::hdfs::directory{ $yarn_nm_app_log_dir:
+          hdp-hadoop::hdfs::directory{ $yarn_nm_app_log_dir:
+            service_state => $service_state,
+            owner => $yarn_user,
+            mode  => '744',
+            recursive_chmod => true
+          }
+        }
+      }
+
+
+      if ($hdp::params::hs_host != "") {
+        $mapred_user = $hdp::params::mapred_user
+        $mapreduce_jobhistory_intermediate_done_dir = $hdp::params::mapreduce_jobhistory_intermediate_done_dir
+        $mapreduce_jobhistory_done_dir = $hdp::params::mapreduce_jobhistory_done_dir
+
+        hdp-hadoop::hdfs::directory{ $mapreduce_jobhistory_intermediate_done_dir:
+          service_state => $service_state,
+          owner => $mapred_user,
+          mode  => '777',
+          recursive_chmod => true
+        }
+
+        hdp-hadoop::hdfs::directory{ $mapreduce_jobhistory_done_dir:
           service_state => $service_state,
-          owner => $yarn_user,
-          mode  => '744',
+          owner => $mapred_user,
+          mode  => '750',
           recursive_chmod => true
         }
       }

ambari-agent/src/main/puppet/modules/hdp-hadoop/manifests/params.pp

@@ -79,6 +79,10 @@ class hdp-hadoop::params(
   
   $mapreduce_libs_path = hdp_default("mapreduce_libs_path","/usr/lib/hadoop-mapreduce/*")
   
+  $mapred_log_dir_prefix = hdp_default("mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+
+  $mapred_pid_dir_prefix = hdp_default("mapreduce_libs_path","/var/run/hadoop-mapreduce")
+
   ### compression related
   if (($hdp::params::lzo_enabled == true) and ($hdp::params::snappy_enabled == true)) {
     $mapred_compress_map_output = true

ambari-agent/src/main/puppet/modules/hdp-hadoop/templates/hadoop-env.sh.erb

@@ -62,6 +62,8 @@ export HADOOP_SSH_OPTS="-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR"
 # Where log files are stored.  $HADOOP_HOME/logs by default.
 export HADOOP_LOG_DIR=<%=scope.function_hdp_template_var("hdfs_log_dir_prefix")%>/$USER
 
+# History server logs
+export HADOOP_MAPRED_LOG_DIR=<%=scope.function_hdp_template_var("mapred_log_dir_prefix")%>/$USER
 
 # Where log files are stored in the secure data environment.
 export HADOOP_SECURE_DN_LOG_DIR=<%=scope.function_hdp_template_var("hdfs_log_dir_prefix")%>/$HADOOP_SECURE_DN_USER
@@ -81,6 +83,9 @@ export HADOOP_SECURE_DN_LOG_DIR=<%=scope.function_hdp_template_var("hdfs_log_dir
 export HADOOP_PID_DIR=<%=scope.function_hdp_template_var("hadoop_pid_dir_prefix")%>/$USER
 export HADOOP_SECURE_DN_PID_DIR=<%=scope.function_hdp_template_var("hadoop_pid_dir_prefix")%>/$HADOOP_SECURE_DN_USER
 
+# History server pid
+export HADOOP_MAPRED_PID_DIR=<%=scope.function_hdp_template_var("mapred_pid_dir_prefix")%>/$USER
+
 # A string representing this instance of hadoop. $USER by default.
 export HADOOP_IDENT_STRING=$USER
 

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/historyserver.pp

@@ -0,0 +1,48 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+class hdp-yarn::historyserver(
+  $service_state = $hdp::params::cluster_service_state,
+  $opts = {}
+) inherits hdp-yarn::params
+{
+  $mapred_user = $hdp-yarn::params::mapred_user
+  
+  if ($service_state == 'no_op') {
+  } elsif ($service_state in 'installed_and_configured') {
+  
+    include hdp-yarn::initialize
+
+    ##Process package
+    hdp-yarn::package{'mapreduce-historyserver':}
+
+  } elsif ($service_state in ['running','stopped']) {
+
+    include hdp-yarn::initialize
+ 
+    hdp-yarn::service{ 'historyserver':
+      ensure       => $service_state,
+      user         => $mapred_user
+    }
+
+  } else {
+    hdp_fail("TODO not implemented yet: service_state = ${service_state}")
+  }
+}

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/init.pp

@@ -22,13 +22,17 @@
 class hdp-yarn::initialize()
 {
   $yarn_user = $hdp-yarn::params::yarn_user
+  $mapred_user = $hdp-yarn::params::mapred_user
   
   ##Process package
   hdp-yarn::package{'yarn-common':}
   
-  # Create user
+  # Create yarn user
   hdp::user { $yarn_user:}
   
+  # Create mapred user
+  hdp::user { $mapred_user:}
+  
   #Generate common configs
   hdp-yarn::generate_common_configs{'yarn-common-configs':}
   

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/params.pp

@@ -19,12 +19,12 @@
 #
 #
 class hdp-yarn::params(
-) inherits hdp::params 
+) inherits hdp-hadoop::params 
 {
 
   $conf_dir = $hdp::params::yarn_conf_dir 
-  
-  ## yarn-env
+    
+  ## yarn-env 
   $hadoop_libexec_dir = hdp_default("yarn/yarn-env/hadoop_libexec_dir","/usr/lib/hadoop/libexec")
   
   $hadoop_common_home = hdp_default("yarn/yarn-env/hadoop_common_home","/usr/lib/hadoop")

ambari-agent/src/main/puppet/modules/hdp-yarn/manifests/service.pp

@@ -28,15 +28,25 @@ define hdp-yarn::service(
 {
 
   $security_enabled = $hdp::params::security_enabled
-  $log_dir = "${hdp-yarn::params::yarn_log_dir_prefix}"
-  $pid_dir = "${hdp-yarn::params::yarn_pid_dir_prefix}/${user}"
-  $yarn_daemon = "${hdp::params::yarn_bin}/yarn-daemon.sh"
+  
+  if ($name == 'historyserver') {
+    $log_dir = "${hdp-yarn::params::mapred_log_dir_prefix}"
+    $pid_dir = "${hdp-yarn::params::mapred_pid_dir_prefix}/${user}"
+    $daemon = "${hdp::params::mapred_bin}/mr-jobhistory-daemon.sh"
+    $pid_file = "${pid_dir}/mapred-${user}-${name}.pid"
+  } else {
+    $log_dir = "${hdp-yarn::params::yarn_log_dir_prefix}"
+    $pid_dir = "${hdp-yarn::params::yarn_pid_dir_prefix}/${user}"
+    $daemon = "${hdp::params::yarn_bin}/yarn-daemon.sh"
+    $pid_file = "${pid_dir}/yarn-${user}-${name}.pid"
+  }
+  
   $hadoop_libexec_dir = $hdp-yarn::params::hadoop_libexec_dir
    
-  $cmd = "export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir} && ${yarn_daemon} --config ${hdp-yarn::params::conf_dir}"
+  $cmd = "export HADOOP_LIBEXEC_DIR=${hadoop_libexec_dir} && ${daemon} --config ${hdp-yarn::params::conf_dir}"
   
   
-  $pid_file = "${pid_dir}/yarn-${user}-${name}.pid"
+
   
   
   

ambari-agent/src/main/puppet/modules/hdp/manifests/configfile.pp

@@ -29,6 +29,9 @@ define hdp::configfile(
   $namenode_host = $hdp::params::namenode_host,
   $jtnode_host = $hdp::params::jtnode_host,
   $snamenode_host = $hdp::params::snamenode_host,
+  $rm_host = $hdp::params::rm_host,
+  $nm_hosts = $hdp::params::nm_hosts,
+  $hs_host = $hdp::params::hs_host,
   $slave_hosts = $hdp::params::slave_hosts,
   $mapred_tt_hosts = $hdp::params::mapred_tt_hosts,
   $all_hosts = $hdp::params::all_hosts,
@@ -46,6 +49,9 @@ define hdp::configfile(
   $gateway_host = $hdp::params::gateway_host,
   $public_namenode_host = $hdp::params::public_namenode_host,
   $public_snamenode_host = $hdp::params::public_snamenode_host,
+  $public_rm_host = $hdp::params::public_rm_host,
+  $public_nm_hosts = $hdp::params::public_nm_hosts,
+  $public_hs_host = $hdp::params::public_hs_host,
   $public_jtnode_host = $hdp::params::public_jtnode_host,
   $public_hbase_master_hosts = $hdp::params::public_hbase_master_hosts,
   $public_zookeeper_hosts = $hdp::params::public_zookeeper_hosts,

ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp

@@ -62,6 +62,10 @@ class hdp::params()
   $jtnode_host = hdp_default("jtnode_host")
   $slave_hosts = hdp_default("slave_hosts")
 
+  $rm_host = hdp_default("rm_host")
+  $nm_hosts = hdp_default("nm_hosts")
+  $hs_host = hdp_default("hs_host")
+
   $zookeeper_hosts = hdp_default("zookeeper_hosts")
 
   $hbase_master_hosts = hdp_default("hbase_master_hosts", "")
@@ -127,6 +131,9 @@ class hdp::params()
   if ($hostAttributes != undef) {
     $public_namenode_host = hdp_host_attribute($hostAttributes,"publicfqdn",$namenode_host)
     $public_snamenode_host = hdp_host_attribute($hostAttributes,"publicfqdn",$snamenode_host)
+    $public_rm_host = hdp_host_attribute($hostAttributes,"publicfqdn",$rm_host)
+    $public_nm_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$nm_hosts)
+    $public_hs_host = hdp_host_attribute($hostAttributes,"publicfqdn",$hs_host)
     $public_jtnode_host = hdp_host_attribute($hostAttributes,"publicfqdn",$jtnode_host)
     $public_hbase_master_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$hbase_master_hosts)
     $public_zookeeper_hosts = hdp_host_attribute($hostAttributes,"publicfqdn",$zookeeper_hosts)
@@ -139,6 +146,9 @@ class hdp::params()
   } else {
     $public_namenode_host = hdp_default("namenode_host")
     $public_snamenode_host = hdp_default("snamenode_host")
+    $public_rm_host = hdp_default("rm_host")
+    $public_nm_hosts = hdp_default("nm_hosts")
+    $public_hs_host = hdp_default("hs_host")
     $public_jtnode_host = hdp_default("jtnode_host")
     $public_hbase_master_hosts = hdp_default("hbase_master_hosts")
     $public_zookeeper_hosts = hdp_default("zookeeper_hosts")
@@ -192,11 +202,15 @@ class hdp::params()
   $hive_apps_whs_dir = hdp_default("hive_apps_whs_dir", "/apps/hive/warehouse")
   $webhcat_apps_dir = hdp_default("webhcat_apps_dir", "/apps/webhcat")
   $hbase_hdfs_root_dir = hdp_default("hbase-site/hbase.hdfs.root.dir","/apps/hbase/data")
-  
+
   $yarn_nm_app_log_dir = hdp_default("yarn-site/yarn.nodemanager.remote-app-log-dir","/app-logs")
-  
+
   $yarn_log_aggregation_enabled = hdp_default("yarn-site/yarn.log-aggregation-enable","true")
 
+  $mapreduce_jobhistory_intermediate_done_dir = hdp_default("mapred-site/mapreduce.jobhistory.intermediate-done-dir","/mr-history/tmp")
+  
+  $mapreduce_jobhistory_done_dir = hdp_default("mapred-site/mapreduce.jobhistory.done-dir","/mr-history/done")
+  
   $user_group = hdp_default("user_group","hadoop")
 
   $ganglia_enabled = hdp_default("ganglia_enabled",true) 
@@ -291,6 +305,7 @@ class hdp::params()
       $hadoop_bin = "/usr/lib/hadoop/bin"
     }
     $yarn_bin = "/usr/lib/hadoop-yarn/sbin"
+    $mapred_bin = "/usr/lib/hadoop-mapreduce/sbin"
     $hadoop_conf_dir = "/etc/hadoop/conf"
     $yarn_conf_dir = "/etc/hadoop/conf"
     $zk_conf_dir = "/etc/zookeeper/conf"
@@ -413,6 +428,14 @@ class hdp::params()
       }
     },
 
+    mapreduce-historyserver => { 
+      'ALL' => {
+        64 => {
+          'ALL' => ['hadoop-mapreduce-historyserver']
+        }
+      }
+    },
+
     lzo => {
       'ALL' => {
         'ALL' => {

ambari-server/src/main/java/org/apache/ambari/server/metadata/RoleCommandOrder.java

@@ -86,6 +86,18 @@ public class RoleCommandOrder {
   public static void initialize() {
     addDependency(Role.SECONDARY_NAMENODE, RoleCommand.START, Role.NAMENODE,
         RoleCommand.START);
+    addDependency(Role.RESOURCEMANAGER, RoleCommand.START, Role.NAMENODE,
+        RoleCommand.START);
+    addDependency(Role.RESOURCEMANAGER, RoleCommand.START, Role.DATANODE,
+        RoleCommand.START);
+    addDependency(Role.NODEMANAGER, RoleCommand.START, Role.NAMENODE,
+        RoleCommand.START);
+    addDependency(Role.NODEMANAGER, RoleCommand.START, Role.DATANODE,
+        RoleCommand.START);
+    addDependency(Role.HISTORYSERVER, RoleCommand.START, Role.NAMENODE,
+        RoleCommand.START);
+    addDependency(Role.HISTORYSERVER, RoleCommand.START, Role.DATANODE,
+        RoleCommand.START);
     addDependency(Role.HBASE_MASTER, RoleCommand.START, Role.ZOOKEEPER_SERVER,
         RoleCommand.START);
     addDependency(Role.HBASE_MASTER, RoleCommand.START, Role.NAMENODE,
@@ -136,6 +148,8 @@ public class RoleCommandOrder {
         Role.JOBTRACKER, RoleCommand.START);
     addDependency(Role.MAPREDUCE_SERVICE_CHECK, RoleCommand.EXECUTE,
         Role.TASKTRACKER, RoleCommand.START);
+    addDependency(Role.RESOURCEMANAGER_SERVICE_CHECK, RoleCommand.EXECUTE,
+        Role.RESOURCEMANAGER, RoleCommand.START);
     addDependency(Role.OOZIE_SERVICE_CHECK, RoleCommand.EXECUTE,
         Role.OOZIE_SERVER, RoleCommand.START);
     addDependency(Role.WEBHCAT_SERVICE_CHECK, RoleCommand.EXECUTE,
@@ -177,10 +191,22 @@ public class RoleCommandOrder {
         Role.JOBTRACKER, RoleCommand.STOP);
     addDependency(Role.NAMENODE, RoleCommand.STOP,
         Role.TASKTRACKER, RoleCommand.STOP);
+    addDependency(Role.NAMENODE, RoleCommand.STOP,
+        Role.RESOURCEMANAGER, RoleCommand.STOP);
+    addDependency(Role.NAMENODE, RoleCommand.STOP,
+        Role.NODEMANAGER, RoleCommand.STOP);
+    addDependency(Role.NAMENODE, RoleCommand.STOP,
+        Role.HISTORYSERVER, RoleCommand.STOP);
     addDependency(Role.DATANODE, RoleCommand.STOP,
         Role.JOBTRACKER, RoleCommand.STOP);
     addDependency(Role.DATANODE, RoleCommand.STOP,
         Role.TASKTRACKER, RoleCommand.STOP);
+    addDependency(Role.DATANODE, RoleCommand.STOP,
+        Role.RESOURCEMANAGER, RoleCommand.STOP);
+    addDependency(Role.DATANODE, RoleCommand.STOP,
+        Role.NODEMANAGER, RoleCommand.STOP);
+    addDependency(Role.DATANODE, RoleCommand.STOP,
+        Role.HISTORYSERVER, RoleCommand.STOP);
 
     addDependency(Role.SECONDARY_NAMENODE, RoleCommand.UPGRADE,
         Role.NAMENODE, RoleCommand.UPGRADE);

ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java

@@ -86,6 +86,9 @@ public class StageUtils {
     componentToClusterInfoKeyMap.put("NAMENODE", "namenode_host");
     componentToClusterInfoKeyMap.put("JOBTRACKER", "jtnode_host");
     componentToClusterInfoKeyMap.put("SNAMENODE", "snamenode_host");
+    componentToClusterInfoKeyMap.put("RESOURCEMANAGER", "rm_host");
+    componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts");
+    componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host");
     componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts");
     componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts");
     componentToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "hbase_rs_hosts");

ambari-server/src/main/resources/stacks/HDP/2.0.1/services/HDFS/configuration/global.xml

@@ -36,6 +36,21 @@
     <value></value>
     <description>Secondary NameNode.</description>
   </property>
+  <property>
+    <name>rm_host</name>
+    <value></value>
+    <description>Resource Manager.</description>
+  </property>
+  <property>
+    <name>nm_hosts</name>
+    <value></value>
+    <description>List of Node Manager Hosts.</description>
+  </property>
+  <property>
+    <name>hs_host</name>
+    <value></value>
+    <description>History Server.</description>
+  </property>
   <property>
     <name>fs_checkpoint_dir</name>
     <value>/hadoop/hdfs/namesecondary</value>

ambari-server/src/main/resources/stacks/HDP/2.0.1/services/MAPREDUCEv2/configuration/capacity-scheduler.xml

@@ -1,112 +0,0 @@
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<configuration>
-
-  <property>
-    <name>yarn.scheduler.capacity.maximum-applications</name>
-    <value>10000</value>
-    <description>
-      Maximum number of applications that can be pending and running.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
-    <value>0.1</value>
-    <description>
-      Maximum percent of resources in the cluster which can be used to run 
-      application masters i.e. controls number of concurrent running
-      applications.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.queues</name>
-    <value>default</value>
-    <description>
-      The queues at the this level (root is the root queue).
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.capacity</name>
-    <value>100</value>
-    <description>
-      The total capacity as a percentage out of 100 for this queue.
-      If it has child queues then this includes their capacity as well.
-      The child queues capacity should add up to their parent queue's capacity
-      or less.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.capacity</name>
-    <value>100</value>
-    <description>Default queue target capacity.</description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
-    <value>1</value>
-    <description>
-      Default queue user limit a percentage from 0.0 to 1.0.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
-    <value>100</value>
-    <description>
-      The maximum capacity of the default queue. 
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.state</name>
-    <value>RUNNING</value>
-    <description>
-      The state of the default queue. State can be one of RUNNING or STOPPED.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.acl_submit_jobs</name>
-    <value>*</value>
-    <description>
-      The ACL of who can submit jobs to the default queue.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
-    <value>*</value>
-    <description>
-      The ACL of who can administer jobs on the default queue.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.acl_administer_queues</name>
-    <value>*</value>
-    <description>
-      The ACL for who can administer this queue i.e. change sub-queue 
-      allocations.
-    </description>
-  </property>
-
-</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.1/services/MAPREDUCEv2/configuration/mapred-site.xml

@@ -528,4 +528,22 @@ user)</description>
   <description> Comma separated list of queues configured for this jobtracker.</description>
 </property>
 
+<property>
+  <name>mapreduce.shuffle.port</name>
+  <value>8081</value>
+  <description>Default port that the ShuffleHandler will run on. ShuffleHandler is a service run at the NodeManager to facilitate transfers of intermediate Map outputs to requesting Reducers.</description>
+</property>
+
+<property>
+  <name>mapreduce.jobhistory.intermediate-done-dir</name>
+  <value>/mr-history/tmp</value>
+  <description>Directory where history files are written by MapReduce jobs.</description>
+</property>
+
+<property>
+  <name>mapreduce.jobhistory.done-dir</name>
+  <value>/mr-history/done</value>
+  <description>Directory where history files are managed by the MR JobHistory Server.</description>
+</property>
+
 </configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.1/services/MAPREDUCEv2/configuration/yarn-site.xml

@@ -1,156 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<!-- ResourceManager -->
-
-  <property>
-    <name>yarn.resourcemanager.resource-tracker.address</name>
-    <value>TODO-RMNODE-HOSTNAME:8025</value>
-  </property>
-
-  <property>
-    <name>yarn.resourcemanager.scheduler.address</name>
-    <value>TODO-RMNODE-HOSTNAME:8030</value>
-  </property>
-  
-  <property>
-    <name>yarn.resourcemanager.address</name>
-    <value>TODO-RMNODE-HOSTNAME:8050</value>
-  </property>
-
-	<property>
-    <name>yarn.resourcemanager.admin.address</name>
-    <value>TODO-RMNODE-HOSTNAME:8141</value>
-	</property>
-
-  <property>
-   <name>yarn.resourcemanager.scheduler.class</name>
-   <value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.minimum-allocation-mb</name>
-    <value>1024</value>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.maximum-allocation-mb</name>
-    <value>8192</value>
-  </property>
-
-<!-- NodeManager -->
-
-  <property>
-    <name>yarn.nodemanager.address</name>
-    <value>0.0.0.0:45454</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.local-dirs</name>
-    <value>TODO-YARN-LOCAL-DIR</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.resource.memory-mb</name>
-    <value>8192</value>
-    <description>Amount of physical memory, in MB, that can be allocated
-      for containers.</description>
-  </property>
-
-  <property>
-    <name>yarn.application.classpath</name>
-    <value>/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*</value>
-  <description>Classpath for typical applications.</description>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.vmem-pmem-ratio</name>
-    <value>2.1</value>
-    <description>Ratio between virtual memory to physical memory when
-    setting memory limits for containers. Container allocations are
-    expressed in terms of physical memory, and virtual memory usage
-    is allowed to exceed this allocation by this ratio.
-    </description>
-  </property>
-  
-  <property>
-    <name>yarn.nodemanager.container-executor.class</name>
-    <value>org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor</value>
-    <description>ContainerExecutor for launching containers</description>
-  </property>
- 
-  <property>
-    <name>yarn.nodemanager.aux-services</name>
-    <value>mapreduce.shuffle</value>
-    <description>Auxilliary services of NodeManager</description>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.aux-services.\1.class</name>
-    <value>org.apache.hadoop.mapred.ShuffleHandler</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.log-dirs</name>
-    <value>TODO-YARN-LOG-DIR</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.container-monitor.interval-ms</name>
-    <value>3000</value>
-    <description>The interval, in milliseconds, for which the node manager
-    waits  between two cycles of monitoring its containers' memory usage. 
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.health-checker.script.path</name>
-    <value>/etc/hadoop/conf/health_check</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.health-checker.interval-ms</name>
-    <value>135000</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.health-checker.script.timeout-ms</name>
-    <value>60000</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.log.retain-second</name>
-    <value>604800</value>
-  </property>
-
-  <property>
-    <name>yarn.log-aggregation-enable</name>
-    <value>true</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.remote-app-log-dir</name>
-    <value>/app-logs</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
-    <value>logs</value>
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.log-aggregation.compression-type</name>
-    <value>gz</value> 
-  </property>
-
-  <property>
-    <name>yarn.nodemanager.delete.debug-delay-sec</name>
-    <value>36000</value>
-  </property>
-
-
-</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.1/services/YARN/configuration/mapred-queue-acls.xml

@@ -1,39 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- mapred-queue-acls.xml -->
-<configuration>
-
-
-<!-- queue default -->
-
-  <property>
-    <name>mapred.queue.default.acl-submit-job</name>
-    <value>*</value>
-  </property>
-
-  <property>
-    <name>mapred.queue.default.acl-administer-jobs</name>
-    <value>*</value>
-  </property>
-
-  <!-- END ACLs -->
-
-</configuration>

ambari-server/src/main/resources/stacks/HDP/2.0.1/services/YARN/configuration/mapred-site.xml

@@ -1,537 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-
-<!--
-   Licensed to the Apache Software Foundation (ASF) under one or more
-   contributor license agreements.  See the NOTICE file distributed with
-   this work for additional information regarding copyright ownership.
-   The ASF licenses this file to You under the Apache License, Version 2.0
-   (the "License"); you may not use this file except in compliance with
-   the License.  You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration xmlns:xi="http://www.w3.org/2001/XInclude">
-
-<!-- i/o properties -->
-
-  <property>
-    <name>io.sort.mb</name>
-    <value></value>
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>io.sort.record.percent</name>
-    <value>.2</value>
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>io.sort.spill.percent</name>
-    <value></value>
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>io.sort.factor</name>
-    <value>100</value>
-    <description>No description</description>
-  </property>
-
-<!-- map/reduce properties -->
-
-<property>
-  <name>mapred.tasktracker.tasks.sleeptime-before-sigkill</name>
-  <value>250</value>
-  <description>Normally, this is the amount of time before killing
-  processes, and the recommended-default is 5.000 seconds - a value of
-  5000 here.  In this case, we are using it solely to blast tasks before
-  killing them, and killing them very quickly (1/4 second) to guarantee
-  that we do not leave VMs around for later jobs.
-  </description>
-</property>
-
-  <property>
-    <name>mapred.job.tracker.handler.count</name>
-    <value>50</value>
-    <description>
-    The number of server threads for the JobTracker. This should be roughly
-    4% of the number of tasktracker nodes.
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.system.dir</name>
-    <value>/mapred/system</value>
-    <description>No description</description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker</name>
-    <!-- cluster variant -->
-    <value></value>
-    <description>No description</description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapred.job.tracker.http.address</name>
-    <!-- cluster variant -->
-    <value></value>
-    <description>No description</description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <!-- cluster specific -->
-    <name>mapred.local.dir</name>
-    <value></value>
-    <description>No description</description>
-    <final>true</final>
-  </property>
-
-  <property>
-  <name>mapreduce.cluster.administrators</name>
-  <value> hadoop</value>
-  </property>
-
-  <property>
-    <name>mapred.reduce.parallel.copies</name>
-    <value>30</value>
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>mapred.tasktracker.map.tasks.maximum</name>
-    <value></value>
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>mapred.tasktracker.reduce.tasks.maximum</name>
-    <value></value>
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>tasktracker.http.threads</name>
-    <value>50</value>
-  </property>
-
-  <property>
-    <name>mapred.map.tasks.speculative.execution</name>
-    <value>false</value>
-    <description>If true, then multiple instances of some map tasks
-               may be executed in parallel.</description>
-  </property>
-
-  <property>
-    <name>mapred.reduce.tasks.speculative.execution</name>
-    <value>false</value>
-    <description>If true, then multiple instances of some reduce tasks
-               may be executed in parallel.</description>
-  </property>
-
-  <property>
-    <name>mapred.reduce.slowstart.completed.maps</name>
-    <value>0.05</value>
-  </property>
-
-  <property>
-    <name>mapred.inmem.merge.threshold</name>
-    <value>1000</value>
-    <description>The threshold, in terms of the number of files
-  for the in-memory merge process. When we accumulate threshold number of files
-  we initiate the in-memory merge and spill to disk. A value of 0 or less than
-  0 indicates we want to DON'T have any threshold and instead depend only on
-  the ramfs's memory consumption to trigger the merge.
-  </description>
-  </property>
-
-  <property>
-    <name>mapred.job.shuffle.merge.percent</name>
-    <value>0.66</value>
-    <description>The usage threshold at which an in-memory merge will be
-  initiated, expressed as a percentage of the total memory allocated to
-  storing in-memory map outputs, as defined by
-  mapred.job.shuffle.input.buffer.percent.
-  </description>
-  </property>
-
-  <property>
-    <name>mapred.job.shuffle.input.buffer.percent</name>
-    <value>0.7</value>
-    <description>The percentage of memory to be allocated from the maximum heap
-  size to storing map outputs during the shuffle.
-  </description>
-  </property>
-
-  <property>
-    <name>mapred.map.output.compression.codec</name>
-    <value></value>
-    <description>If the map outputs are compressed, how should they be
-      compressed
-    </description>
-  </property>
-
-<property>
-  <name>mapred.output.compression.type</name>
-  <value>BLOCK</value>
-  <description>If the job outputs are to compressed as SequenceFiles, how should
-               they be compressed? Should be one of NONE, RECORD or BLOCK.
-  </description>
-</property>
-
-
-  <property>
-    <name>mapred.jobtracker.completeuserjobs.maximum</name>
-    <value>5</value>
-  </property>
-
-  <property>
-    <name>mapred.jobtracker.taskScheduler</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapred.jobtracker.restart.recover</name>
-    <value>false</value>
-    <description>"true" to enable (job) recovery upon restart,
-               "false" to start afresh
-    </description>
-  </property>
-
-  <property>
-    <name>mapred.job.reduce.input.buffer.percent</name>
-    <value>0.0</value>
-    <description>The percentage of memory- relative to the maximum heap size- to
-  retain map outputs during the reduce. When the shuffle is concluded, any
-  remaining map outputs in memory must consume less than this threshold before
-  the reduce can begin.
-  </description>
-  </property>
-
- <property>
-  <name>mapreduce.reduce.input.limit</name>
-  <value>10737418240</value>
-  <description>The limit on the input size of the reduce. (This value
-  is 10 Gb.)  If the estimated input size of the reduce is greater than
-  this value, job is failed. A value of -1 means that there is no limit
-  set. </description>
-</property>
-
-
-  <!-- copied from kryptonite configuration -->
-  <property>
-    <name>mapred.compress.map.output</name>
-    <value></value>
-  </property>
-
-
-  <property>
-    <name>mapred.task.timeout</name>
-    <value>600000</value>
-    <description>The number of milliseconds before a task will be
-  terminated if it neither reads an input, writes an output, nor
-  updates its status string.
-  </description>
-  </property>
-
-  <property>
-    <name>jetty.connector</name>
-    <value>org.mortbay.jetty.nio.SelectChannelConnector</value>
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>mapred.task.tracker.task-controller</name>
-    <value></value>
-   <description>
-     TaskController which is used to launch and manage task execution.
-  </description>
-  </property>
-
-  <property>
-    <name>mapred.child.root.logger</name>
-    <value>INFO,TLA</value>
-  </property>
-
-  <property>
-    <name>mapred.child.java.opts</name>
-    <value></value>
-
-    <description>No description</description>
-  </property>
-
-  <property>
-    <name>mapred.cluster.map.memory.mb</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapred.cluster.reduce.memory.mb</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapred.job.map.memory.mb</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapred.job.reduce.memory.mb</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapred.cluster.max.map.memory.mb</name>
-    <value></value>
-  </property>
-
-  <property>
-    <name>mapred.cluster.max.reduce.memory.mb</name>
-    <value></value>
-  </property>
-
-<property>
-  <name>mapred.hosts</name>
-  <value></value>
-</property>
-
-<property>
-  <name>mapred.hosts.exclude</name>
-  <value></value>
-</property>
-
-<property>
-  <name>mapred.max.tracker.blacklists</name>
-  <value>16</value>
-  <description>
-    if node is reported blacklisted by 16 successful jobs within timeout-window, it will be graylisted
-  </description>
-</property>
-
-<property>
-  <name>mapred.healthChecker.script.path</name>
-  <value></value>
-</property>
-
-<property>
-  <name>mapred.healthChecker.interval</name>
-  <value>135000</value>
-</property>
-
-<property>
-  <name>mapred.healthChecker.script.timeout</name>
-  <value>60000</value>
-</property>
-
-<property>
-  <name>mapred.job.tracker.persist.jobstatus.active</name>
-  <value>false</value>
-  <description>Indicates if persistency of job status information is
-  active or not.
-  </description>
-</property>
-
-<property>
-  <name>mapred.job.tracker.persist.jobstatus.hours</name>
-  <value>1</value>
-  <description>The number of hours job status information is persisted in DFS.
-    The job status information will be available after it drops of the memory
-    queue and between jobtracker restarts. With a zero value the job status
-    information is not persisted at all in DFS.
-  </description>
-</property>
-
-<property>
-  <name>mapred.job.tracker.persist.jobstatus.dir</name>
-  <value></value>
-  <description>The directory where the job status information is persisted
-   in a file system to be available after it drops of the memory queue and
-   between jobtracker restarts.
-  </description>
-</property>
-
-<property>
-  <name>mapred.jobtracker.retirejob.check</name>
-  <value>10000</value>
-</property>
-
-<property>
-  <name>mapred.jobtracker.retirejob.interval</name>
-  <value>0</value>
-</property>
-
-<property>
-  <name>mapred.job.tracker.history.completed.location</name>
-  <value>/mapred/history/done</value>
-  <description>No description</description>
-</property>
-
-<property>
-  <name>mapred.task.maxvmem</name>
-  <value></value>
-  <final>true</final>
-   <description>No description</description>
-</property>
-
-<property>
-  <name>mapred.jobtracker.maxtasks.per.job</name>
-  <value></value>
-  <final>true</final>
-  <description>The maximum number of tasks for a single job.
-  A value of -1 indicates that there is no maximum.  </description>
-</property>
-
-<property>
-  <name>mapreduce.fileoutputcommitter.marksuccessfuljobs</name>
-  <value>false</value>
-</property>
-
-<property>
-  <name>mapred.userlog.retain.hours</name>
-  <value></value>
-</property>
-
-<property>
-  <name>mapred.job.reuse.jvm.num.tasks</name>
-  <value>1</value>
-  <description>
-    How many tasks to run per jvm. If set to -1, there is no limit
-  </description>
-  <final>true</final>
-</property>
-
-<property>
-  <name>mapreduce.jobtracker.kerberos.principal</name>
-  <value></value>
-  <description>
-      JT user name key.
- </description>
-</property>
-
-<property>
-  <name>mapreduce.tasktracker.kerberos.principal</name>
-   <value></value>
-  <description>
-       tt user name key. "_HOST" is replaced by the host name of the task tracker.
-   </description>
-</property>
-
-
-  <property>
-    <name>hadoop.job.history.user.location</name>
-    <value>none</value>
-    <final>true</final>
-  </property>
-
-
- <property>
-   <name>mapreduce.jobtracker.keytab.file</name>
-   <value></value>
-   <description>
-       The keytab for the jobtracker principal.
-   </description>
-
-</property>
-
- <property>
-   <name>mapreduce.tasktracker.keytab.file</name>
-   <value></value>
-    <description>The filename of the keytab for the task tracker</description>
- </property>
-
- <property>
-   <name>mapreduce.jobtracker.staging.root.dir</name>
-   <value>/user</value>
- <description>The Path prefix for where the staging directories should be placed. The next level is always the user's
-   name. It is a path in the default file system.</description>
- </property>
-
- <property>
-      <name>mapreduce.tasktracker.group</name>
-      <value>hadoop</value>
-      <description>The group that the task controller uses for accessing the task controller. The mapred user must be a member and users should *not* be members.</description>
-
- </property>
-
-  <property>
-    <name>mapreduce.jobtracker.split.metainfo.maxsize</name>
-    <value>50000000</value>
-    <final>true</final>
-     <description>If the size of the split metainfo file is larger than this, the JobTracker will fail the job during
-    initialize.
-   </description>
-  </property>
-  <property>
-    <name>mapreduce.history.server.embedded</name>
-    <value>false</value>
-    <description>Should job history server be embedded within Job tracker
-process</description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapreduce.history.server.http.address</name>
-    <!-- cluster variant -->
-    <value></value>
-    <description>Http address of the history server</description>
-    <final>true</final>
-  </property>
-
-  <property>
-    <name>mapreduce.jobhistory.kerberos.principal</name>
-    <!-- cluster variant -->
-  <value></value>
-    <description>Job history user name key. (must map to same user as JT
-user)</description>
-  </property>
-
- <property>
-   <name>mapreduce.jobhistory.keytab.file</name>
-    <!-- cluster variant -->
-   <value></value>
-   <description>The keytab for the job history server principal.</description>
- </property>
-
-<property>
-  <name>mapred.jobtracker.blacklist.fault-timeout-window</name>
-  <value>180</value>
-  <description>
-    3-hour sliding window (value is in minutes)
-  </description>
-</property>
-
-<property>
-  <name>mapred.jobtracker.blacklist.fault-bucket-width</name>
-  <value>15</value>
-  <description>
-    15-minute bucket size (value is in minutes)
-  </description>
-</property>
-
-<property>
-  <name>mapred.queue.names</name>
-  <value>default</value>
-  <description> Comma separated list of queues configured for this jobtracker.</description>
-</property>
-
-<property>
-  <name>mapreduce.shuffle.port</name>
-  <value>8081</value>
-  <description>Default port that the ShuffleHandler will run on. ShuffleHandler is a service run at the NodeManager to facilitate transfers of intermediate Map outputs to requesting Reducers.</description>
-</property>
-
-</configuration>