HADOOP-3475. Fix MapTask to correctly size the accounting allocation of

io.sort.mb.




git-svn-id: https://svn.apache.org/repos/asf/hadoop/core/branches/branch-0.17@668861 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -17,6 +17,8 @@ Release 0.17.1
     HADOOP-3477. Fix build to not package contrib/*/bin twice in
     distributions.  (Adam Heath via cutting)
 
+    HADOOP-3475. Fix MapTask to correctly size the accounting allocation of
+    io.sort.mb. (cdouglas)
 
 Release 0.17.1 - Unreleased
 

src/java/org/apache/hadoop/mapred/MapTask.java

@@ -310,6 +310,9 @@ class MapTask extends Task {
     private static final int KEYSTART = 1;  // key offset in acct
     private static final int VALSTART = 2;  // val offset in acct
     private static final int ACCTSIZE = 3;  // total #fields in acct
+    private static final int RECSIZE =
+                       (ACCTSIZE + 1) * 4;  // acct bytes per record
+
 
     // spill accounting
     private volatile int numSpills = 0;
@@ -352,12 +355,12 @@ class MapTask extends Task {
       // buffers and accounting
       int maxMemUsage = sortmb << 20;
       int recordCapacity = (int)(maxMemUsage * recper);
-      recordCapacity += (recordCapacity >>> 2) % 4;
+      recordCapacity -= recordCapacity % RECSIZE;
       kvbuffer = new byte[maxMemUsage - recordCapacity];
       bufvoid = kvbuffer.length;
-      int kvcapacity = recordCapacity >>> 2;
-      kvoffsets = new int[kvcapacity];
-      kvindices = new int[recordCapacity - kvcapacity];
+      recordCapacity /= RECSIZE;
+      kvoffsets = new int[recordCapacity];
+      kvindices = new int[recordCapacity * ACCTSIZE];
       softBufferLimit = (int)(kvbuffer.length * spillper);
       softRecordLimit = (int)(kvoffsets.length * spillper);
       // k/v serialization