MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. Contributed by Arun C. Murthy.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1@1506114 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -27,6 +27,8 @@ Release 1.3.0 - unreleased
     MAPREDUCE-5406. Improve logging around Task Tracker exiting with JVM manager
     inconsistent state. (Chelsey Chang via cnauroth)
 
+    MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. (acmurthy)
+
   BUG FIXES
 
     MAPREDUCE-5047. keep.failed.task.files=true causes job failure on 

src/mapred/mapred-default.xml

@@ -588,6 +588,22 @@
   </description>
 </property>
 
+<property>
+  <name>mapred.map.child.log.level</name>
+  <value>INFO</value>
+  <description>The logging level for the map task. The allowed levels are:
+  OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+  </description>
+</property>
+
+<property>
+  <name>mapred.reduce.child.log.level</name>
+  <value>INFO</value>
+  <description>The logging level for the reduce task. The allowed levels are:
+  OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+  </description>
+</property>
+
 <property>
   <name>mapred.inmem.merge.threshold</name>
   <value>1000</value>

src/mapred/org/apache/hadoop/mapred/JobConf.java

@@ -45,6 +45,7 @@ import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.util.ClassUtil;
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.Tool;
+import org.apache.log4j.Level;
 
 /** 
  * A map/reduce job configuration.
@@ -336,6 +337,29 @@ public class JobConf extends Configuration {
 
   private Credentials credentials = new Credentials();
   
+  /**
+   * Configuration key to set the logging {@link Level} for the map task.
+   *
+   * The allowed logging levels are:
+   * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+   */
+  public static final String MAPRED_MAP_TASK_LOG_LEVEL = 
+    "mapred.map.child.log.level";
+  
+  /**
+   * Configuration key to set the logging {@link Level} for the reduce task.
+   *
+   * The allowed logging levels are:
+   * OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL.
+   */
+  public static final String MAPRED_REDUCE_TASK_LOG_LEVEL = 
+    "mapred.reduce.child.log.level";
+  
+  /**
+   * Default logging level for map/reduce tasks.
+   */
+  public static final Level DEFAULT_LOG_LEVEL = Level.INFO;
+  
   /**
    * Construct a map/reduce job configuration.
    */

src/mapred/org/apache/hadoop/mapred/MapTaskRunner.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
 import java.io.*;
 
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.log4j.Level;
 
 /** Runs a map task. */
 class MapTaskRunner extends TaskRunner {
@@ -69,4 +70,11 @@ class MapTaskRunner extends TaskRunner {
     return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV, super.getChildEnv(jobConf));
   }
 
+  @Override
+  public Level getLogLevel(JobConf jobConf) {
+    return Level.toLevel(jobConf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, 
+                                     JobConf.DEFAULT_LOG_LEVEL.toString()), 
+                         Level.INFO);
+  }
+
 }

src/mapred/org/apache/hadoop/mapred/ReduceTaskRunner.java

@@ -20,6 +20,7 @@ package org.apache.hadoop.mapred;
 import java.io.*;
 
 import org.apache.hadoop.mapred.TaskTracker.TaskInProgress;
+import org.apache.log4j.Level;
 
 /** Runs a reduce task. */
 class ReduceTaskRunner extends TaskRunner {
@@ -73,4 +74,11 @@ class ReduceTaskRunner extends TaskRunner {
                        super.getChildEnv(jobConf));
   }
 
+  @Override
+  public Level getLogLevel(JobConf jobConf) {
+    return Level.toLevel(jobConf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, 
+                                     JobConf.DEFAULT_LOG_LEVEL.toString()),
+                         Level.INFO);
+  }
+
 }

src/mapred/org/apache/hadoop/mapred/TaskRunner.java

@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.util.Shell;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.log4j.Level;
 
 /** Base class that runs a task in a separate process.  Tasks are run in a
  * separate process in order to isolate the map/reduce system code from bugs in
@@ -183,6 +184,13 @@ abstract class TaskRunner extends Thread {
     return jobConf.get(JobConf.MAPRED_TASK_ENV);
   }
   
+  /**
+   * Get the log {@link Level} for the child map/reduce tasks.
+   * @param jobConf
+   * @return the log-level for the child map/reduce tasks
+   */
+  public abstract Level getLogLevel(JobConf jobConf);
+  
   @Override
   public final void run() {
     String errorInfo = "Child Error";
@@ -477,7 +485,7 @@ abstract class TaskRunner extends Thread {
     vargs.add("-Dlog4j.configuration=task-log4j.properties");
     vargs.add("-Dhadoop.log.dir=" + 
         new File(System.getProperty("hadoop.log.dir")).getAbsolutePath());
-    vargs.add("-Dhadoop.root.logger=INFO,TLA");
+    vargs.add("-Dhadoop.root.logger=" + getLogLevel(conf).toString() + ",TLA");
     vargs.add("-Dhadoop.tasklog.taskid=" + taskid);
     vargs.add("-Dhadoop.tasklog.iscleanup=" + t.isTaskCleanupTask());
     vargs.add("-Dhadoop.tasklog.totalLogFileSize=" + logSize);

src/test/org/apache/hadoop/mapreduce/TestChild.java

@@ -29,6 +29,7 @@ import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.HadoopTestCase;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.log4j.Level;
 
 public class TestChild extends HadoopTestCase {
   private static String TEST_ROOT_DIR =
@@ -68,6 +69,12 @@ public class TestChild extends HadoopTestCase {
                      mapJavaOpts, 
                      mapJavaOpts, MAP_OPTS_VAL);
       }
+      
+      Level logLevel = 
+        Level.toLevel(conf.get(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, 
+                               Level.INFO.toString()));  
+      assertEquals(JobConf.MAPRED_MAP_TASK_LOG_LEVEL + "has value of " + 
+                   logLevel, logLevel, Level.OFF);
     }
   }
   
@@ -94,6 +101,12 @@ public class TestChild extends HadoopTestCase {
                      reduceJavaOpts, 
                      reduceJavaOpts, REDUCE_OPTS_VAL);
       }
+      
+      Level logLevel = 
+        Level.toLevel(conf.get(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, 
+                               Level.INFO.toString()));  
+      assertEquals(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL + "has value of " + 
+                   logLevel, logLevel, Level.OFF);
     }
   }
   
@@ -108,6 +121,9 @@ public class TestChild extends HadoopTestCase {
       conf.set(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, REDUCE_OPTS_VAL);
     }
     
+    conf.set(JobConf.MAPRED_MAP_TASK_LOG_LEVEL, Level.OFF.toString());
+    conf.set(JobConf.MAPRED_REDUCE_TASK_LOG_LEVEL, Level.OFF.toString());
+    
     Job job = MapReduceTestUtil.createJob(conf, inDir, outDir, 
                 numMaps, numReds);
     job.setMapperClass(MyMapper.class);