Ver código fonte

MAPREDUCE-5420. Remove mapreduce.task.tmp.dir from mapred-default.xml. Contributed by James Carman. (harsh)

Harsh J 10 anos atrás
pai
commit
cb99f43305

+ 3 - 0
hadoop-mapreduce-project/CHANGES.txt

@@ -235,6 +235,9 @@ Release 2.7.0 - UNRELEASED
 
   IMPROVEMENTS
 
+    MAPREDUCE-5420. Remove mapreduce.task.tmp.dir from mapred-default.xml
+    (James Carman via harsh)
+
     MAPREDUCE-5932. Provide an option to use a dedicated reduce-side shuffle
     log (Gera Shegalov via jlowe)
 

+ 0 - 2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java

@@ -166,8 +166,6 @@ public interface MRJobConfig {
 
   public static final String PRESERVE_FILES_PATTERN = "mapreduce.task.files.preserve.filepattern";
 
-  public static final String TASK_TEMP_DIR = "mapreduce.task.tmp.dir";
-
   public static final String TASK_DEBUGOUT_LINES = "mapreduce.task.debugout.lines";
 
   public static final String RECORDS_BEFORE_PROGRESS = "mapreduce.task.merge.progress.records";

+ 0 - 2
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java

@@ -274,8 +274,6 @@ public class ConfigUtil {
         MRJobConfig.PRESERVE_FAILED_TASK_FILES),
       new DeprecationDelta("keep.task.files.pattern",
         MRJobConfig.PRESERVE_FILES_PATTERN),
-      new DeprecationDelta("mapred.child.tmp",
-        MRJobConfig.TASK_TEMP_DIR),
       new DeprecationDelta("mapred.debug.out.lines",
         MRJobConfig.TASK_DEBUGOUT_LINES),
       new DeprecationDelta("mapred.merge.recordsBeforeProgress",

+ 0 - 12
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml

@@ -319,18 +319,6 @@
   </description>
 </property>
 
-<property>
-  <name>mapreduce.task.tmp.dir</name>
-  <value>./tmp</value>
-  <description> To set the value of tmp directory for map and reduce tasks.
-  If the value is an absolute path, it is directly assigned. Otherwise, it is
-  prepended with task's working directory. The java tasks are executed with
-  option -Djava.io.tmpdir='the absolute path of the tmp dir'. Pipes and
-  streaming are set with environment variable,
-   TMPDIR='the absolute path of the tmp dir'
-  </description>
-</property>
-
 <property>
   <name>mapreduce.map.log.level</name>
   <value>INFO</value>

+ 0 - 1
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/resources/job_1329348432655_0001_conf.xml

@@ -166,7 +166,6 @@
 <property><!--Loaded from mapred-default.xml--><name>mapreduce.job.end-notification.max.attempts</name><value>5</value></property>
 <property><!--Loaded from job.xml--><name>mapreduce.jobhistory.max-age-ms</name><value>10000000000</value></property>
 <property><!--Loaded from job.xml--><name>yarn.resourcemanager.zookeeper-store.session.timeout-ms</name><value>60000</value></property>
-<property><!--Loaded from job.xml--><name>mapreduce.task.tmp.dir</name><value>./tmp</value></property>
 <property><!--Loaded from job.xml--><name>dfs.default.chunk.view.size</name><value>32768</value></property>
 <property><!--Loaded from job.xml--><name>kfs.bytes-per-checksum</name><value>512</value></property>
 <property><!--Loaded from job.xml--><name>mapreduce.reduce.memory.mb</name><value>512</value></property>

+ 1 - 30
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRChildTask.java

@@ -391,36 +391,7 @@ public class TestMiniMRChildTask {
       ioe.printStackTrace();           
     }
   }
-  
-  /**
-   * Tests task's temp directory.
-   * 
-   * In this test, we give different values to mapreduce.task.tmp.dir
-   * both relative and absolute. And check whether the temp directory 
-   * is created. We also check whether java.io.tmpdir value is same as 
-   * the directory specified. We create a temp file and check if is is 
-   * created in the directory specified.
-   */
-  @Test
-  public void testTaskTempDir(){
-    try {
-      JobConf conf = new JobConf(mr.getConfig());
-      
-      // intialize input, output directories
-      Path inDir = new Path("testing/wc/input");
-      Path outDir = new Path("testing/wc/output");
-      String input = "The input";
-      configure(conf, inDir, outDir, input, 
-          MapClass.class, IdentityReducer.class);
-      launchTest(conf, inDir, outDir, input);
-      
-    } catch(Exception e) {
-      e.printStackTrace();
-      fail("Exception in testing temp dir");
-      tearDown();
-    }
-  }
-
+ 
   /**
    * To test OS dependent setting of default execution path for a MapRed task.
    * Mainly that we can use MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV to set -