HADOOP-4129. Changed memory limits of TaskTracker and Tasks to be in KiloBytes rather than bytes. Contributed by Vinod Kumar Vavilapalli.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/trunk@696002 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -562,6 +562,9 @@ Trunk (unreleased changes)
     HADOOP-3570. Includes user specified libjar files in the client side 
     classpath path. (Sharad Agarwal via ddas)
 
+    HADOOP-4129. Changed memory limits of TaskTracker and Tasks to be in
+    KiloBytes rather than bytes. (Vinod Kumar Vavilapalli via acmurthy)
+
 Release 0.18.1 - 2008-09-17
 
   IMPROVEMENTS

conf/hadoop-default.xml

@@ -1424,22 +1424,22 @@ creations/deletions), or "all".</description>
 <property>
   <name>mapred.tasktracker.tasks.maxmemory</name>
   <value>-1</value>
-  <description> The maximum amount of virtual memory all tasks running on a
-    tasktracker, including sub-processes they launch, can use. This value is
-    used to compute the amount of free memory available for tasks. Any task
-    scheduled on this tasktracker is guaranteed and constrained to use a 
-    share of this amount. Any task exceeding its share will be killed. 
-    If set to -1, this functionality is disabled, and mapred.task.maxmemory
-    is ignored.
+  <description> The maximum amount of virtual memory in kilobytes all tasks 
+  	running on a tasktracker, including sub-processes they launch, can use. 
+  	This value is used to compute the amount of free memory available for 
+  	tasks. Any task scheduled on this tasktracker is guaranteed and constrained
+  	 to use a share of this amount. Any task exceeding its share will be 
+  	killed. If set to -1, this functionality is disabled, and 
+  	mapred.task.maxmemory is ignored.
   </description>
 </property>
 
 <property>
   <name>mapred.task.maxmemory</name>
   <value>-1</value>
-  <description> The maximum amount of memory any task of a job will use.
-    A task of this job will be scheduled on a tasktracker, only if the 
-    amount of free memory on the tasktracker is greater than or 
+  <description> The maximum amount of memory in kilobytes any task of a job 
+    will use. A task of this job will be scheduled on a tasktracker, only if 
+    the amount of free memory on the tasktracker is greater than or 
     equal to this value. If set to -1, tasks are assured a memory limit on
     the tasktracker equal to 
     mapred.tasktracker.tasks.maxmemory/number of slots. If the value of 

src/core/org/apache/hadoop/util/ProcfsBasedProcessTree.java

@@ -189,7 +189,7 @@ public class ProcfsBasedProcessTree {
    * Get the cumulative virtual memory used by all the processes in the
    * process-tree.
    * 
-   * @return cumulative virtual memory used by the process-tree in bytes.
+   * @return cumulative virtual memory used by the process-tree in kilobytes.
    */
   public long getCumulativeVmem() {
     long total = 0;
@@ -198,7 +198,7 @@ public class ProcfsBasedProcessTree {
         total += p.getVmem();
       }
     }
-    return total;
+    return total/1024;
   }
 
   /**
@@ -466,4 +466,4 @@ public class ProcfsBasedProcessTree {
       return children;
     }
   }
-}
+}

src/mapred/org/apache/hadoop/mapred/JobConf.java

@@ -1331,7 +1331,7 @@ public class JobConf extends Configuration {
    * If set to {@link #DISABLED_VIRTUAL_MEMORY_LIMIT}, this functionality 
    * is disabled.
    * 
-   * @return maximum amount of virtual memory to divide among
+   * @return maximum amount of virtual memory in kilobytes to divide among
    * @see #getMaxVirtualMemoryForTask()
    */
   public long getMaxVirtualMemoryForTasks() {
@@ -1343,7 +1343,7 @@ public class JobConf extends Configuration {
    * Set the maximum amount of virtual memory all tasks running on a
    * tasktracker, including sub-processes they launch, can use.
    * 
-   * @param vmem maximum amount of virtual memory that can be used.
+   * @param vmem maximum amount of virtual memory in kilobytes that can be used.
    * @see #getMaxVirtualMemoryForTasks()
    */
   public void setMaxVirtualMemoryForTasks(long vmem) {
@@ -1363,7 +1363,7 @@ public class JobConf extends Configuration {
    * mapred.tasktracker.tasks.maxmemory is set to -1, this value is 
    * ignored.
    * 
-   * @return The maximum amount of memory any task of this job will use.
+   * @return The maximum amount of memory any task of this job will use, in kilobytes.
    * @see #getMaxVirtualMemoryForTasks()
    */
   public long getMaxVirtualMemoryForTask() {
@@ -1373,7 +1373,8 @@ public class JobConf extends Configuration {
   /**
    * Set the maximum amount of memory any task of this job can use.
    * 
-   * @param vmem Maximum amount of memory any task of this job can use.
+   * @param vmem Maximum amount of memory in kilobytes any task of this job 
+   * can use.
    * @see #getMaxVirtualMemoryForTask()
    */
   public void setMaxVirtualMemoryForTask(long vmem) {

src/mapred/org/apache/hadoop/mapred/TaskMemoryManagerThread.java

@@ -196,7 +196,7 @@ class TaskMemoryManagerThread extends Thread {
         long currentMemUsage = pTree.getCumulativeVmem();
         long limit = ptInfo.getMemLimit();
         LOG.info("Memory usage of ProcessTree " + pId + " :" + currentMemUsage
-            + ". Limit : " + limit);
+            + "kB. Limit : " + limit + "kB");
 
         if (limit != JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT
             && currentMemUsage > limit) {
@@ -204,7 +204,7 @@ class TaskMemoryManagerThread extends Thread {
           // Clean up.
           String msg = "TaskTree [pid=" + pId + ",tipID=" + tid
               + "] is running beyond memory-limits. Current usage : "
-              + currentMemUsage + ". Limit : " + limit + ". Killing task.";
+              + currentMemUsage + "kB. Limit : " + limit + "kB. Killing task.";
           LOG.warn(msg);
           taskTracker.cleanUpOverMemoryTask(tid, msg);
 

src/mapred/org/apache/hadoop/mapred/TaskTracker.java

@@ -1192,7 +1192,7 @@ public class TaskTracker
   /**
    * Return the maximum amount of memory available for all tasks on 
    * this tracker
-   * @return maximum amount of virtual memory
+   * @return maximum amount of virtual memory in kilobytes
    */
   long getMaxVirtualMemoryForTasks() {
     return maxVirtualMemoryForTasks;
@@ -1208,7 +1208,7 @@ public class TaskTracker
    * and the total amount of maximum virtual memory that can be
    * used by all currently running tasks.
    * 
-   * @return amount of free virtual memory that can be assured for
+   * @return amount of free virtual memory in kilobytes that can be assured for
    * new tasks
    */
   private synchronized long findFreeVirtualMemory() {

src/mapred/org/apache/hadoop/mapred/TaskTrackerStatus.java

@@ -67,7 +67,7 @@ class TaskTrackerStatus implements Writable {
     /**
      * Set the amount of free virtual memory that is available for running
      * a new task
-     * @param freeVMem amount of free virtual memory
+     * @param freeVMem amount of free virtual memory in kilobytes
      */
     void setFreeVirtualMemory(long freeVmem) {
       freeVirtualMemory = freeVmem;
@@ -80,7 +80,7 @@ class TaskTrackerStatus implements Writable {
      * If this is {@link JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT}, it should 
      * be ignored and not used in computation.
      * 
-     *@return amount of free virtual memory.
+     *@return amount of free virtual memory in kilobytes.
      */
     long getFreeVirtualMemory() {
       return freeVirtualMemory;
@@ -88,7 +88,7 @@ class TaskTrackerStatus implements Writable {
 
     /**
      * Set the default amount of virtual memory per task.
-     * @param vmem amount of free virtual memory.
+     * @param vmem amount of free virtual memory in kilobytes.
      */
     void setDefaultVirtualMemoryPerTask(long defaultVmem) {
       defaultVirtualMemoryPerTask = defaultVmem;
@@ -102,7 +102,7 @@ class TaskTrackerStatus implements Writable {
      * {@link JobConf.DISABLED_VIRTUAL_MEMORY_LIMIT}, it should be ignored 
      * and not used in any computation.
      * 
-     * @return default amount of virtual memory per task. 
+     * @return default amount of virtual memory per task in kilobytes. 
      */    
     long getDefaultVirtualMemoryPerTask() {
       return defaultVirtualMemoryPerTask;

src/test/org/apache/hadoop/mapred/TestTaskTrackerMemoryManager.java

@@ -98,7 +98,7 @@ public class TestTaskTrackerMemoryManager extends TestCase {
 
     Pattern diagMsgPattern = Pattern
         .compile("TaskTree \\[pid=[0-9]*,tipID=.*\\] is running beyond "
-            + "memory-limits. Current usage : [0-9]*. Limit : [0-9]*. Killing task.");
+            + "memory-limits. Current usage : [0-9]*kB. Limit : [0-9]*kB. Killing task.");
     Matcher mat = null;
 
     // Start cluster with proper configuration.
@@ -172,8 +172,8 @@ public class TestTaskTrackerMemoryManager extends TestCase {
     long PER_TASK_LIMIT = 444; // Enough to kill off WordCount.
     Pattern diagMsgPattern = Pattern
         .compile("TaskTree \\[pid=[0-9]*,tipID=.*\\] is running beyond "
-            + "memory-limits. Current usage : [0-9]*. Limit : "
-            + PER_TASK_LIMIT + ". Killing task.");
+            + "memory-limits. Current usage : [0-9]*kB. Limit : "
+            + PER_TASK_LIMIT + "kB. Killing task.");
     Matcher mat = null;
 
     // Start cluster with proper configuration.
@@ -235,4 +235,4 @@ public class TestTaskTrackerMemoryManager extends TestCase {
       }
     }
   }
-}
+}

src/test/org/apache/hadoop/util/TestProcfsBasedProcessTree.java

@@ -39,7 +39,7 @@ public class TestProcfsBasedProcessTree extends TestCase {
   private String shellScript;
   private static final int N = 10; // Controls the RogueTask
 
-  private static final int memoryLimit = 15000000; // bytes
+  private static final int memoryLimit = 15000; // kilobytes
   private static final long PROCESSTREE_RECONSTRUCTION_INTERVAL =
     ProcfsBasedProcessTree.DEFAULT_SLEEPTIME_BEFORE_SIGKILL; // msec
 
@@ -125,7 +125,7 @@ public class TestProcfsBasedProcessTree extends TestCase {
       while (true) {
         LOG.info("ProcessTree: " + p.toString());
         long mem = p.getCumulativeVmem();
-        LOG.info("Memory usage: " + mem + "bytes.");
+        LOG.info("Memory usage: " + mem + "kB.");
         if (mem > memoryLimit) {
           p.destroy();
           break;
@@ -146,4 +146,4 @@ public class TestProcfsBasedProcessTree extends TestCase {
       LOG.info("Interrupted while joining RogueTaskThread.");
     }
   }
-}
+}