Merge -c 1306741 from branch-1 to branch-1.0 to fix MAPREDUCE-3377. Ensure OutputCommitter.checkOutputSpecs is called prior to copying job.xml.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.0@1306744 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -22,6 +22,9 @@ Release 1.0.3 - unreleased
     HDFS-3127. Do not throw exceptions when FSImage.restoreStorageDirs() failes.
     (Brandon Li via szetszwo)
 
+    MAPREDUCE-3377. Ensure OutputCommitter.checkOutputSpecs is called prior to
+    copying job.xml. (Jane Chen via acmurthy)
+
 Release 1.0.2 - 2012.03.24
 
   NEW FEATURES

build.xml

@@ -1592,14 +1592,6 @@
 
     <copy todir="${dist.dir}/" file="build.xml"/>
 
-    <subant target="task-controller">
-      <fileset dir="." includes="build.xml"/>
-    </subant>
-
-    <subant target="jsvc">
-      <fileset dir="." includes="build.xml"/>
-    </subant>
-
     <chmod perm="ugo+x" type="file" parallel="false">
         <fileset dir="${dist.dir}/bin"/>
         <fileset dir="${dist.dir}/sbin"/>
@@ -1768,14 +1760,6 @@
       </fileset>
     </copy>
 
-    <subant target="task-controller">
-      <fileset dir="." includes="build.xml"/>
-    </subant>
-
-    <subant target="jsvc">
-      <fileset dir="." includes="build.xml"/>
-    </subant>
-
     <chmod perm="ugo+x" type="file" parallel="false">
         <fileset dir="${dist.dir}/bin"/>
         <fileset dir="${dist.dir}/libexec"/>

src/mapred/org/apache/hadoop/mapred/JobClient.java

@@ -878,8 +878,6 @@ public class JobClient extends Configured implements MRConstants, Tool  {
           }
           JobContext context = new JobContext(jobCopy, jobId);
 
-          jobCopy = (JobConf)context.getConfiguration();
-
           // Check the output specification
           if (reduces == 0 ? jobCopy.getUseNewMapper() : 
             jobCopy.getUseNewReducer()) {
@@ -890,6 +888,8 @@ public class JobClient extends Configured implements MRConstants, Tool  {
           } else {
             jobCopy.getOutputFormat().checkOutputSpecs(fs, jobCopy);
           }
+          
+          jobCopy = (JobConf)context.getConfiguration();
 
           // Create the splits for the job
           FileSystem fs = submitJobDir.getFileSystem(jobCopy);

src/test/org/apache/hadoop/mapreduce/TestMROutputFormat.java

@@ -0,0 +1,188 @@
+package org.apache.hadoop.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.JobConf;
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+public class TestMROutputFormat {
+
+  @Test
+  public void testJobSubmission() throws Exception {
+    JobConf conf = new JobConf();
+    Job job = new Job(conf);
+    job.setInputFormatClass(TestInputFormat.class);
+    job.setMapperClass(TestMapper.class);
+    job.setOutputFormatClass(TestOutputFormat.class);
+    job.setOutputKeyClass(IntWritable.class);
+    job.setOutputValueClass(IntWritable.class);
+    job.waitForCompletion(true);
+    assertTrue(job.isSuccessful());
+  }
+  
+  public static class TestMapper
+  extends Mapper<IntWritable, IntWritable, IntWritable, IntWritable> {
+    public void map(IntWritable key, IntWritable value, Context context) 
+    throws IOException, InterruptedException {
+      context.write(key, value);
+    }
+  }
+}
+
+class TestInputFormat extends InputFormat<IntWritable, IntWritable> {
+
+  @Override
+  public RecordReader<IntWritable, IntWritable> createRecordReader(
+      InputSplit split, TaskAttemptContext context) throws IOException,
+      InterruptedException {
+    return new RecordReader<IntWritable, IntWritable>() {
+
+      private boolean done = false;
+      
+      @Override
+      public void close() throws IOException {
+      }
+
+      @Override
+      public IntWritable getCurrentKey() throws IOException,
+          InterruptedException {
+	return new IntWritable(0);
+      }
+
+      @Override
+      public IntWritable getCurrentValue() throws IOException,
+          InterruptedException {
+	return new IntWritable(0);
+      }
+
+      @Override
+      public float getProgress() throws IOException, InterruptedException {
+	return done ? 0 : 1;
+      }
+
+      @Override
+      public void initialize(InputSplit split, TaskAttemptContext context)
+          throws IOException, InterruptedException {
+      }
+
+      @Override
+      public boolean nextKeyValue() throws IOException, InterruptedException {
+	if (!done) {
+	  done = true;
+	  return true;
+	}
+	return false;
+      }
+    };
+  }
+
+  @Override
+  public List<InputSplit> getSplits(JobContext context) throws IOException,
+      InterruptedException {    
+    List<InputSplit> list = new ArrayList<InputSplit>();
+    list.add(new TestInputSplit());
+    return list;
+  }
+}
+
+class TestInputSplit extends InputSplit implements Writable {
+  
+  @Override
+  public long getLength() throws IOException, InterruptedException {
+	return 1;
+  }
+
+  @Override
+  public String[] getLocations() throws IOException, InterruptedException {
+	String[] hosts = {"localhost"};
+	return hosts;
+  }
+
+  @Override
+  public void readFields(DataInput in) throws IOException {
+  }
+
+  @Override
+  public void write(DataOutput out) throws IOException {
+  }	
+}
+
+class TestOutputFormat extends OutputFormat<IntWritable, IntWritable> 
+implements Configurable {
+
+  public static final String TEST_CONFIG_NAME = "mapred.test.jobsubmission";
+  private Configuration conf;
+  
+  @Override
+  public void checkOutputSpecs(JobContext context) throws IOException,
+      InterruptedException {
+    conf.setBoolean(TEST_CONFIG_NAME, true);
+  }
+
+  @Override
+  public OutputCommitter getOutputCommitter(TaskAttemptContext context)
+      throws IOException, InterruptedException {
+    return new OutputCommitter() {
+
+      @Override
+      public void abortTask(TaskAttemptContext taskContext) throws IOException {
+      }
+
+      @Override
+      public void commitTask(TaskAttemptContext taskContext) throws IOException {
+      }
+
+      @Override
+      public boolean needsTaskCommit(TaskAttemptContext taskContext)
+          throws IOException {
+	return false;
+      }
+
+      @Override
+      public void setupJob(JobContext jobContext) throws IOException {
+      }
+
+      @Override
+      public void setupTask(TaskAttemptContext taskContext) throws IOException {
+      }
+    };
+  }
+
+  @Override
+  public RecordWriter<IntWritable, IntWritable> getRecordWriter(
+      TaskAttemptContext context) throws IOException, InterruptedException {
+    assertTrue(context.getConfiguration().getBoolean(TEST_CONFIG_NAME, false));
+    return new RecordWriter<IntWritable, IntWritable>() {
+
+      @Override
+      public void close(TaskAttemptContext context) throws IOException,
+          InterruptedException {	
+      }
+
+      @Override
+      public void write(IntWritable key, IntWritable value) throws IOException,
+          InterruptedException {	
+      }
+    }; 
+  }
+  
+  @Override
+  public Configuration getConf() {
+      return conf;
+  }
+
+  @Override
+  public void setConf(Configuration conf) {
+      this.conf = conf;        
+  }
+}