Browse Source

MAPREDUCE-5911. Terasort TeraOutputFormat does not check for output directory existance. Contributed by Bruno P. Kinoshita.

Ivan Mitic 10 years ago
parent
commit
7bbda6ef92

+ 3 - 0
hadoop-mapreduce-project/CHANGES.txt

@@ -438,6 +438,9 @@ Release 2.6.0 - UNRELEASED
     MAPREDUCE-5542. Killing a job just as it finishes can generate an NPE in
     MAPREDUCE-5542. Killing a job just as it finishes can generate an NPE in
     client (Rohith via jlowe)
     client (Rohith via jlowe)
 
 
+    MAPREDUCE-5911. Terasort TeraOutputFormat does not check for output
+    directory existance (Bruno P. Kinoshita via ivanmi)
+
 Release 2.5.1 - 2014-09-05
 Release 2.5.1 - 2014-09-05
 
 
   INCOMPATIBLE CHANGES
   INCOMPATIBLE CHANGES

+ 6 - 0
hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraOutputFormat.java

@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileAlreadyExistsException;
 import org.apache.hadoop.mapred.InvalidJobConfException;
 import org.apache.hadoop.mapred.InvalidJobConfException;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.OutputCommitter;
 import org.apache.hadoop.mapreduce.OutputCommitter;
@@ -90,6 +91,11 @@ public class TeraOutputFormat extends FileOutputFormat<Text,Text> {
     // get delegation token for outDir's file system
     // get delegation token for outDir's file system
     TokenCache.obtainTokensForNamenodes(job.getCredentials(),
     TokenCache.obtainTokensForNamenodes(job.getCredentials(),
         new Path[] { outDir }, job.getConfiguration());
         new Path[] { outDir }, job.getConfiguration());
+
+    if (outDir.getFileSystem(job.getConfiguration()).exists(outDir)) {
+      throw new FileAlreadyExistsException("Output directory " + outDir + 
+                                           " already exists");
+    }
   }
   }
 
 
   public RecordWriter<Text,Text> getRecordWriter(TaskAttemptContext job
   public RecordWriter<Text,Text> getRecordWriter(TaskAttemptContext job