MAPREDUCE-4399. Change the Jetty response buffer size to improve shuffle performance. Contributed by Luke Lu.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-1.0@1365209 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -9,6 +9,9 @@ Release 1.0.4 - Unreleased
     HADOOP-7154. Should set MALLOC_ARENA_MAX in hadoop-env.sh
     (todd via mattf)
  
+    MAPREDUCE-4399. Change the Jetty response buffer size to improve 
+    shuffle performance. (Luke Lu via suresh)
+
   BUG FIXES
 
     HDFS-3652. FSEditLog failure removes the wrong edit stream when storage

src/mapred/org/apache/hadoop/mapred/TaskTracker.java

@@ -3790,7 +3790,8 @@ public class TaskTracker implements MRConstants, TaskUmbilicalProtocol,
   public static class MapOutputServlet extends HttpServlet {
     private static final long serialVersionUID = 1L;
     private static final int MAX_BYTES_TO_READ = 64 * 1024;
-    
+    // work around jetty internal buffering issues
+    private static final int RESPONSE_BUFFER_SIZE = MAX_BYTES_TO_READ + 16;
     private static LRUCache<String, Path> fileCache = new LRUCache<String, Path>(FILE_CACHE_SIZE);
     private static LRUCache<String, Path> fileIndexCache = new LRUCache<String, Path>(FILE_CACHE_SIZE);
     
@@ -3892,10 +3893,9 @@ public class TaskTracker implements MRConstants, TaskUmbilicalProtocol,
         //set the custom "for-reduce-task" http header to the reduce task number
         //for which this map output is being transferred
         response.setHeader(FOR_REDUCE_TASK, Integer.toString(reduce));
-        
-        //use the same buffersize as used for reading the data from disk
-        response.setBufferSize(MAX_BYTES_TO_READ);
-        
+
+        response.setBufferSize(RESPONSE_BUFFER_SIZE);
+
         /**
          * Read the data from the sigle map-output file and
          * send it to the reducer.