HADOOP-7728. Enable task memory management to be configurable in hadoop config setup script. (ramya)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.20-security-205@1190020 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -9,6 +9,9 @@ Release 0.20.205.1 - unreleased
 
     HDFS-2501. Add version prefix and root methods to webhdfs.  (szetszwo)
 
+    HADOOP-7728. Enable task memory management to be configurable in hadoop 
+    config setup script. (ramya)
+
   BUG FIXES
 
     HDFS-2441. Remove the Content-Type set by HttpServer.QuotingInputFilter in

src/packages/hadoop-setup-conf.sh

@@ -55,6 +55,16 @@ usage: $0 <parameters>
      --dfs-support-append=false|true                                 Enable append
      --hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts'    Setup proxy users for hadoop
      --hbase-user=hbase                                              User which hbase is running as. Defaults to hbase
+     --mapred-cluster-map-memory-mb=memory                           Virtual memory of a map slot for the MR framework. Defaults to -1
+     --mapred-cluster-reduce-memory-mb=memory                        Virtual memory, of a reduce slot for the MR framework. Defaults to -1
+     --mapred-cluster-max-map-memory-mb=memory                       Maximum virtual memory of a single map task. Defaults to -1
+                                                                     This value should be set to (mapred.tasktracker.map.tasks.maximum * mapred.cluster.map.memory.mb)
+     --mapred-cluster-max-reduce-memory-mb=memory                    maximum virtual memory of a single reduce task. Defaults to -1
+                                                                     This value should be set to (mapred.tasktracker.reduce.tasks.maximum * mapred.cluster.reduce.memory.mb)
+     --mapred-job-map-memory-mb=memory                               Virtual memory of a single map slot for a job. Defaults to -1
+                                                                     This value should be <= mapred.cluster.max.map.memory.mb
+     --mapred-job-reduce-memory-mb=memory                            Virtual memory, of a single reduce slot for a job. Defaults to -1
+                                                                     This value should be <= mapred.cluster.max.reduce.memory.mb
   "
   exit 1
 }
@@ -199,6 +209,12 @@ OPTS=$(getopt \
   -l 'hadoop-proxy-users:' \
   -l 'dfs-support-append:' \
   -l 'hbase-user:' \
+  -l 'mapred-cluster-map-memory-mb:' \
+  -l 'mapred-cluster-reduce-memory-mb:' \
+  -l 'mapred-cluster-max-map-memory-mb:' \
+  -l 'mapred-cluster-max-reduce-memory-mb:' \
+  -l 'mapred-job-map-memory-mb:' \
+  -l 'mapred-job-reduce-memory-mb:' \
   -o 'h' \
   -- "$@") 
   
@@ -334,6 +350,30 @@ while true ; do
       HBASE_USER=$2; shift 2
       AUTOMATED=1
       ;;
+    --mapred-cluster-map-memory-mb)
+      MAPRED_CLUSTER_MAP_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapred-cluster-reduce-memory-mb)
+      MAPRED_CLUSTER_REDUCE_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapred-cluster-max-map-memory-mb)
+      MAPRED_CLUSTER_MAX_MAP_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapred-cluster-max-reduce-memory-mb)
+      MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapred-job-map-memory-mb)
+      MAPRED_JOB_MAP_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapred-job-reduce-memory-mb)
+      MAPRED_JOB_REDUCE_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
     --)
       shift ; break
       ;;
@@ -365,6 +405,12 @@ HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
 DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
 DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
 HBASE_USER=${HBASE_USER:-hbase}
+MAPRED_CLUSTER_MAP_MEMORY_MB=${MAPRED_CLUSTER_MAP_MEMORY_MB:--1}
+MAPRED_CLUSTER_REDUCE_MEMORY_MB=${MAPRED_CLUSTER_REDUCE_MEMORY_MB:--1} 
+MAPRED_CLUSTER_MAX_MAP_MEMORY_MB=${MAPRED_CLUSTER_MAX_MAP_MEMORY_MB:--1} 
+MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB=${MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB:--1} 
+MAPRED_JOB_MAP_MEMORY_MB=${MAPRED_JOB_MAP_MEMORY_MB=:--1} 
+MAPRED_JOB_REDUCE_MEMORY_MB=${MAPRED_JOB_REDUCE_MEMORY_MB:--1} 
 KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
 HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
 MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}

src/packages/templates/conf/mapred-site.xml

@@ -259,4 +259,80 @@
     <name>mapred.jobtracker.retirejob.interval</name>
     <value>0</value>
   </property>
+
+  <property>
+    <name>mapred.cluster.map.memory.mb</name>
+    <value>${MAPRED_CLUSTER_MAP_MEMORY_MB}</value>
+    <description>The size, in terms of virtual memory, of a single map slot
+      in the Map-Reduce framework, used by the scheduler.
+      A job can ask for multiple slots for a single map task via
+      mapred.job.map.memory.mb, upto the limit specified by
+      mapred.cluster.max.map.memory.mb, if the scheduler supports the feature.
+      The value of -1 indicates that this feature is turned off.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.cluster.reduce.memory.mb</name>
+    <value>${MAPRED_CLUSTER_REDUCE_MEMORY_MB}</value>
+    <description>The size, in terms of virtual memory, of a single reduce slot
+      in the Map-Reduce framework, used by the scheduler.
+      A job can ask for multiple slots for a single reduce task via
+      mapred.job.reduce.memory.mb, upto the limit specified by
+      mapred.cluster.max.reduce.memory.mb, if the scheduler supports the feature.
+      The value of -1 indicates that this feature is turned off.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.cluster.max.map.memory.mb</name>
+    <value>${MAPRED_CLUSTER_MAX_MAP_MEMORY_MB}</value>
+    <description>The maximum size, in terms of virtual memory, of a single map
+      task launched by the Map-Reduce framework, used by the scheduler.
+      A job can ask for multiple slots for a single map task via
+      mapred.job.map.memory.mb, upto the limit specified by
+      mapred.cluster.max.map.memory.mb, if the scheduler supports the feature.
+      The value of -1 indicates that this feature is turned off.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.cluster.max.reduce.memory.mb</name>
+    <value>${MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB}</value>
+    <description>The maximum size, in terms of virtual memory, of a single reduce
+      task launched by the Map-Reduce framework, used by the scheduler.
+      A job can ask for multiple slots for a single reduce task via
+      mapred.job.reduce.memory.mb, upto the limit specified by
+      mapred.cluster.max.reduce.memory.mb, if the scheduler supports the feature.
+      The value of -1 indicates that this feature is turned off.
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.job.map.memory.mb</name>
+    <value>${MAPRED_JOB_MAP_MEMORY_MB}</value>
+    <description>The size, in terms of virtual memory, of a single map task
+      for the job.
+      A job can ask for multiple slots for a single map task, rounded up to the
+      next multiple of mapred.cluster.map.memory.mb and upto the limit
+      specified by mapred.cluster.max.map.memory.mb, if the scheduler supports
+      the feature.
+      The value of -1 indicates that this feature is turned off iff
+      mapred.cluster.map.memory.mb is also turned off (-1).
+    </description>
+  </property>
+
+  <property>
+    <name>mapred.job.reduce.memory.mb</name>
+    <value>${MAPRED_JOB_REDUCE_MEMORY_MB}</value>
+    <description>The size, in terms of virtual memory, of a single reduce task
+      for the job.
+      A job can ask for multiple slots for a single map task, rounded up to the
+      next multiple of mapred.cluster.reduce.memory.mb and upto the limit
+      specified by mapred.cluster.max.reduce.memory.mb, if the scheduler supports
+      the feature.
+      The value of -1 indicates that this feature is turned off iff
+      mapred.cluster.reduce.memory.mb is also turned off (-1).
+    </description>
+  </property>
 </configuration>