HADOOP-7728. Enable task memory management to be configurable in hadoop config setup script. (ramya)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1190035 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-common/CHANGES.txt

@@ -55,6 +55,9 @@ Trunk (unreleased changes)
 
     HADOOP-7758. Make GlobFilter class public. (tucu)
 
+    HADOOP-7728. Enable task memory management to be configurable in hadoop
+    config setup script. (ramya)
+
   BUGS
 
     HADOOP-7606. Upgrade Jackson to version 1.7.1 to match the version required

hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh

@@ -55,6 +55,16 @@ usage: $0 <parameters>
      --dfs-support-append=false|true                                 Enable append
      --hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts'    Setup proxy users for hadoop
      --hbase-user=hbase                                              User which hbase is running as. Defaults to hbase
+     --mapreduce-cluster-mapmemory-mb=memory                         Virtual memory of a map slot for the MR framework. Defaults to -1
+     --mapreduce-cluster-reducememory-mb=memory                      Virtual memory, of a reduce slot for the MR framework. Defaults to -1
+     --mapreduce-jobtracker-maxmapmemory-mb=memory                   Maximum virtual memory of a single map task. Defaults to -1
+                                                                     This value should be set to (mapreduce.cluster.mapmemory.mb * mapreduce.tasktracker.map.tasks.maximum)
+     --mapreduce-jobtracker-maxreducememory-mb=memory                Maximum virtual memory of a single reduce task. Defaults to -1
+                                                                     This value should be set to (mapreduce.cluster.reducememory.mb * mapreduce.tasktracker.reduce.tasks.maximum)
+     --mapreduce-map-memory-mb=memory                                Virtual memory of a single map slot for a job. Defaults to -1
+                                                                     This value should be <= mapred.cluster.max.map.memory.mb
+     --mapreduce-reduce-memory-mb=memory                             Virtual memory, of a single reduce slot for a job. Defaults to -1
+                                                                     This value should be <= mapred.cluster.max.reduce.memory.mb
   "
   exit 1
 }
@@ -199,6 +209,12 @@ OPTS=$(getopt \
   -l 'hadoop-proxy-users:' \
   -l 'dfs-support-append:' \
   -l 'hbase-user:' \
+  -l 'mapreduce-cluster-mapmemory-mb:' \
+  -l 'mapreduce-cluster-reducememory-mb:' \
+  -l 'mapreduce-jobtracker-maxmapmemory-mb:' \
+  -l 'mapreduce-jobtracker-maxreducememory-mb:' \
+  -l 'mapreduce-map-memory-mb:' \
+  -l 'mapreduce-reduce-memory-mb:' \
   -o 'h' \
   -- "$@") 
   
@@ -334,6 +350,30 @@ while true ; do
       HBASE_USER=$2; shift 2
       AUTOMATED=1
       ;;
+    --mapreduce-cluster-mapmemory-mb)
+      MAPREDUCE_CLUSTER_MAPMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-cluster-reducememory-mb)
+      MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-jobtracker-maxmapmemory-mb)
+      MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-jobtracker-maxreducememory-mb)
+      MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-map-memory-mb)
+      MAPREDUCE_MAP_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
+    --mapreduce-reduce-memory-mb)
+      MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
+      AUTOMATED=1
+      ;;
     --)
       shift ; break
       ;;
@@ -365,6 +405,12 @@ HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
 DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
 DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
 HBASE_USER=${HBASE_USER:-hbase}
+MAPREDUCE_CLUSTER_MAPMEMORY_MB=${MAPREDUCE_CLUSTER_MAPMEMORY_MB:--1}
+MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB:--1}
+MAPREDUCE_MAP_MEMORY_MB=${MAPREDUCE_MAP_MEMORY_MB:--1}
+MAPREDUCE_REDUCE_MEMORY_MB=${MAPREDUCE_REDUCE_MEMORY_MB:--1}
 KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
 HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
 MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}

hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml

@@ -280,4 +280,34 @@
     <name>mapred.jobtracker.retirejob.interval</name>
     <value>0</value>
   </property>
+
+  <property>
+    <name>mapreduce.cluster.mapmemory.mb</name>
+    <value>${MAPREDUCE_CLUSTER_MAPMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.cluster.reducememory.mb</name>
+    <value>${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.maxmapmemory.mb</name>
+    <value>${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.jobtracker.maxreducememory.mb</name>
+    <value>${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.map.memory.mb</name>
+    <value>${MAPREDUCE_MAP_MEMORY_MB}</value>
+  </property>
+
+  <property>
+    <name>mapreduce.reduce.memory.mb</name>
+    <value>${MAPREDUCE_REDUCE_MEMORY_MB}</value>
+  </property>
 </configuration>