Make speculative execution optional, since it can break map tasks that have side effects. Also fix TestFileSystem to be safe to use with speculative execution.

git-svn-id: https://svn.apache.org/repos/asf/lucene/hadoop/trunk@378095 13f79535-47bb-0310-9956-ffa450edef68

conf/hadoop-default.xml

@@ -203,6 +203,13 @@
   combining them and writing to disk.</description>
 </property>
 
+<property>
+  <name>mapred.speculative.execution</name>
+  <value>true</value>
+  <description>If true, then multiple instances of some map tasks may
+  be executed in parallel.</description>
+</property>
+
 
 <!-- ipc properties -->
 

src/java/org/apache/hadoop/mapred/TaskInProgress.java

@@ -407,7 +407,8 @@ class TaskInProgress {
         // REMIND - mjc - these constants should be examined
         // in more depth eventually...
         //
-        if (isMapTask() && 
+        if (isMapTask() &&
+            conf.getBoolean("mapred.speculative.execution", true) &&
             (averageProgress - progress >= SPECULATIVE_GAP) &&
             (System.currentTimeMillis() - startTime >= SPECULATIVE_LAG)) {
             return true;

src/test/org/apache/hadoop/fs/TestFileSystem.java

@@ -103,6 +103,9 @@ public class TestFileSystem extends TestCase {
     private byte[] buffer = new byte[BUFFER_SIZE];
     private FileSystem fs;
     private boolean fastCheck;
+
+    // a random suffix per task
+    private String suffix = "-"+random.nextLong();
     
     {
       try {
@@ -131,7 +134,9 @@ public class TestFileSystem extends TestCase {
       random.setSeed(seed);
       reporter.setStatus("creating " + name);
 
-      OutputStream out = fs.create(new File(DATA_DIR, name));
+      // write to temp file initially to permit parallel execution
+      File tempFile = new File(DATA_DIR, name+suffix);
+      OutputStream out = fs.create(tempFile);
 
       long written = 0;
       try {
@@ -150,6 +155,8 @@ public class TestFileSystem extends TestCase {
       } finally {
         out.close();
       }
+      // rename to final location
+      fs.rename(tempFile, new File(DATA_DIR, name));
 
       collector.collect(new UTF8("bytes"), new LongWritable(written));